feat(memory): allow QMD searches via mcporter keep-alive (openclaw#19617) thanks @vignesh07
Verified: - pnpm build - pnpm check - pnpm test:macmini Co-authored-by: vignesh07 <1436853+vignesh07@users.noreply.github.com> Co-authored-by: Tak Hoffman <781889+Takhoffman@users.noreply.github.com>
This commit is contained in:
2
.gitignore
vendored
2
.gitignore
vendored
@@ -99,3 +99,5 @@ package-lock.json
|
|||||||
|
|
||||||
# Local iOS signing overrides
|
# Local iOS signing overrides
|
||||||
apps/ios/LocalSigning.xcconfig
|
apps/ios/LocalSigning.xcconfig
|
||||||
|
# Generated protocol schema (produced via pnpm protocol:gen)
|
||||||
|
dist/protocol.schema.json
|
||||||
|
|||||||
@@ -16,6 +16,7 @@ Docs: https://docs.openclaw.ai
|
|||||||
|
|
||||||
### Fixes
|
### Fixes
|
||||||
|
|
||||||
|
- Memory/QMD: add optional `memory.qmd.mcporter` search routing so QMD `query/search/vsearch` can run through mcporter keep-alive flows (including multi-collection paths) to reduce cold starts, while keeping searches on agent-scoped QMD state for consistent recall. (#19617) Thanks @vignesh07.
|
||||||
- Chat/UI: strip inline reply/audio directive tags (`[[reply_to_current]]`, `[[reply_to:<id>]]`, `[[audio_as_voice]]`) from displayed chat history, live chat event output, and session preview snippets so control tags no longer leak into user-visible surfaces.
|
- Chat/UI: strip inline reply/audio directive tags (`[[reply_to_current]]`, `[[reply_to:<id>]]`, `[[audio_as_voice]]`) from displayed chat history, live chat event output, and session preview snippets so control tags no longer leak into user-visible surfaces.
|
||||||
- Security/Config: block prototype-key traversal during config merge patch and legacy migration merge helpers (`__proto__`, `constructor`, `prototype`) to prevent prototype pollution during config mutation flows. (#22968) Thanks @Clawborn.
|
- Security/Config: block prototype-key traversal during config merge patch and legacy migration merge helpers (`__proto__`, `constructor`, `prototype`) to prevent prototype pollution during config mutation flows. (#22968) Thanks @Clawborn.
|
||||||
- Security/Shell env: validate login-shell executable paths for shell-env fallback (`/etc/shells` + trusted prefixes) and block `SHELL` in dangerous env override policy paths so untrusted shell-path injection falls back safely to `/bin/sh`. Thanks @athuljayaram for reporting.
|
- Security/Shell env: validate login-shell executable paths for shell-env fallback (`/etc/shells` + trusted prefixes) and block `SHELL` in dangerous env override policy paths so untrusted shell-path injection falls back safely to `/bin/sh`. Thanks @athuljayaram for reporting.
|
||||||
|
|||||||
@@ -236,6 +236,13 @@ export const FIELD_HELP: Record<string, string> = {
|
|||||||
"memory.backend": 'Memory backend ("builtin" for OpenClaw embeddings, "qmd" for QMD sidecar).',
|
"memory.backend": 'Memory backend ("builtin" for OpenClaw embeddings, "qmd" for QMD sidecar).',
|
||||||
"memory.citations": 'Default citation behavior ("auto", "on", or "off").',
|
"memory.citations": 'Default citation behavior ("auto", "on", or "off").',
|
||||||
"memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).",
|
"memory.qmd.command": "Path to the qmd binary (default: resolves from PATH).",
|
||||||
|
"memory.qmd.mcporter":
|
||||||
|
"Optional: route QMD searches through mcporter (MCP runtime) instead of spawning `qmd` per query. Intended to avoid per-search cold starts when QMD models are large.",
|
||||||
|
"memory.qmd.mcporter.enabled": "Enable mcporter-backed QMD searches (default: false).",
|
||||||
|
"memory.qmd.mcporter.serverName":
|
||||||
|
"mcporter server name to call (default: qmd). Server should run `qmd mcp` with lifecycle keep-alive.",
|
||||||
|
"memory.qmd.mcporter.startDaemon":
|
||||||
|
"Start `mcporter daemon start` automatically when enabled (default: true).",
|
||||||
"memory.qmd.includeDefaultMemory":
|
"memory.qmd.includeDefaultMemory":
|
||||||
"Whether to automatically index MEMORY.md + memory/**/*.md (default: true).",
|
"Whether to automatically index MEMORY.md + memory/**/*.md (default: true).",
|
||||||
"memory.qmd.paths":
|
"memory.qmd.paths":
|
||||||
|
|||||||
@@ -12,6 +12,7 @@ export type MemoryConfig = {
|
|||||||
|
|
||||||
export type MemoryQmdConfig = {
|
export type MemoryQmdConfig = {
|
||||||
command?: string;
|
command?: string;
|
||||||
|
mcporter?: MemoryQmdMcporterConfig;
|
||||||
searchMode?: MemoryQmdSearchMode;
|
searchMode?: MemoryQmdSearchMode;
|
||||||
includeDefaultMemory?: boolean;
|
includeDefaultMemory?: boolean;
|
||||||
paths?: MemoryQmdIndexPath[];
|
paths?: MemoryQmdIndexPath[];
|
||||||
@@ -21,6 +22,20 @@ export type MemoryQmdConfig = {
|
|||||||
scope?: SessionSendPolicyConfig;
|
scope?: SessionSendPolicyConfig;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type MemoryQmdMcporterConfig = {
|
||||||
|
/**
|
||||||
|
* Route QMD searches through mcporter (MCP runtime) instead of spawning `qmd` per query.
|
||||||
|
* Requires:
|
||||||
|
* - `mcporter` installed and on PATH
|
||||||
|
* - A configured mcporter server that runs `qmd mcp` with `lifecycle: keep-alive`
|
||||||
|
*/
|
||||||
|
enabled?: boolean;
|
||||||
|
/** mcporter server name (defaults to "qmd") */
|
||||||
|
serverName?: string;
|
||||||
|
/** Start the mcporter daemon automatically (defaults to true when enabled). */
|
||||||
|
startDaemon?: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export type MemoryQmdIndexPath = {
|
export type MemoryQmdIndexPath = {
|
||||||
path: string;
|
path: string;
|
||||||
name?: string;
|
name?: string;
|
||||||
|
|||||||
@@ -72,9 +72,18 @@ const MemoryQmdLimitsSchema = z
|
|||||||
})
|
})
|
||||||
.strict();
|
.strict();
|
||||||
|
|
||||||
|
const MemoryQmdMcporterSchema = z
|
||||||
|
.object({
|
||||||
|
enabled: z.boolean().optional(),
|
||||||
|
serverName: z.string().optional(),
|
||||||
|
startDaemon: z.boolean().optional(),
|
||||||
|
})
|
||||||
|
.strict();
|
||||||
|
|
||||||
const MemoryQmdSchema = z
|
const MemoryQmdSchema = z
|
||||||
.object({
|
.object({
|
||||||
command: z.string().optional(),
|
command: z.string().optional(),
|
||||||
|
mcporter: MemoryQmdMcporterSchema.optional(),
|
||||||
searchMode: z.union([z.literal("query"), z.literal("search"), z.literal("vsearch")]).optional(),
|
searchMode: z.union([z.literal("query"), z.literal("search"), z.literal("vsearch")]).optional(),
|
||||||
includeDefaultMemory: z.boolean().optional(),
|
includeDefaultMemory: z.boolean().optional(),
|
||||||
paths: z.array(MemoryQmdPathSchema).optional(),
|
paths: z.array(MemoryQmdPathSchema).optional(),
|
||||||
|
|||||||
@@ -17,7 +17,6 @@ import {
|
|||||||
validateSafeBinArgv,
|
validateSafeBinArgv,
|
||||||
} from "./exec-safe-bin-policy.js";
|
} from "./exec-safe-bin-policy.js";
|
||||||
import { isTrustedSafeBinPath } from "./exec-safe-bin-trust.js";
|
import { isTrustedSafeBinPath } from "./exec-safe-bin-trust.js";
|
||||||
|
|
||||||
export function normalizeSafeBins(entries?: string[]): Set<string> {
|
export function normalizeSafeBins(entries?: string[]): Set<string> {
|
||||||
if (!Array.isArray(entries)) {
|
if (!Array.isArray(entries)) {
|
||||||
return new Set();
|
return new Set();
|
||||||
|
|||||||
@@ -8,6 +8,7 @@ import type {
|
|||||||
MemoryCitationsMode,
|
MemoryCitationsMode,
|
||||||
MemoryQmdConfig,
|
MemoryQmdConfig,
|
||||||
MemoryQmdIndexPath,
|
MemoryQmdIndexPath,
|
||||||
|
MemoryQmdMcporterConfig,
|
||||||
MemoryQmdSearchMode,
|
MemoryQmdSearchMode,
|
||||||
} from "../config/types.memory.js";
|
} from "../config/types.memory.js";
|
||||||
import { resolveUserPath } from "../utils.js";
|
import { resolveUserPath } from "../utils.js";
|
||||||
@@ -50,8 +51,15 @@ export type ResolvedQmdSessionConfig = {
|
|||||||
retentionDays?: number;
|
retentionDays?: number;
|
||||||
};
|
};
|
||||||
|
|
||||||
|
export type ResolvedQmdMcporterConfig = {
|
||||||
|
enabled: boolean;
|
||||||
|
serverName: string;
|
||||||
|
startDaemon: boolean;
|
||||||
|
};
|
||||||
|
|
||||||
export type ResolvedQmdConfig = {
|
export type ResolvedQmdConfig = {
|
||||||
command: string;
|
command: string;
|
||||||
|
mcporter: ResolvedQmdMcporterConfig;
|
||||||
searchMode: MemoryQmdSearchMode;
|
searchMode: MemoryQmdSearchMode;
|
||||||
collections: ResolvedQmdCollection[];
|
collections: ResolvedQmdCollection[];
|
||||||
sessions: ResolvedQmdSessionConfig;
|
sessions: ResolvedQmdSessionConfig;
|
||||||
@@ -79,6 +87,12 @@ const DEFAULT_QMD_LIMITS: ResolvedQmdLimitsConfig = {
|
|||||||
maxInjectedChars: 4_000,
|
maxInjectedChars: 4_000,
|
||||||
timeoutMs: DEFAULT_QMD_TIMEOUT_MS,
|
timeoutMs: DEFAULT_QMD_TIMEOUT_MS,
|
||||||
};
|
};
|
||||||
|
const DEFAULT_QMD_MCPORTER: ResolvedQmdMcporterConfig = {
|
||||||
|
enabled: false,
|
||||||
|
serverName: "qmd",
|
||||||
|
startDaemon: true,
|
||||||
|
};
|
||||||
|
|
||||||
const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = {
|
const DEFAULT_QMD_SCOPE: SessionSendPolicyConfig = {
|
||||||
default: "deny",
|
default: "deny",
|
||||||
rules: [
|
rules: [
|
||||||
@@ -237,6 +251,27 @@ function resolveCustomPaths(
|
|||||||
return collections;
|
return collections;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
function resolveMcporterConfig(raw?: MemoryQmdMcporterConfig): ResolvedQmdMcporterConfig {
|
||||||
|
const parsed: ResolvedQmdMcporterConfig = { ...DEFAULT_QMD_MCPORTER };
|
||||||
|
if (!raw) {
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
if (raw.enabled !== undefined) {
|
||||||
|
parsed.enabled = raw.enabled;
|
||||||
|
}
|
||||||
|
if (typeof raw.serverName === "string" && raw.serverName.trim()) {
|
||||||
|
parsed.serverName = raw.serverName.trim();
|
||||||
|
}
|
||||||
|
if (raw.startDaemon !== undefined) {
|
||||||
|
parsed.startDaemon = raw.startDaemon;
|
||||||
|
}
|
||||||
|
// When enabled, default startDaemon to true.
|
||||||
|
if (parsed.enabled && raw.startDaemon === undefined) {
|
||||||
|
parsed.startDaemon = true;
|
||||||
|
}
|
||||||
|
return parsed;
|
||||||
|
}
|
||||||
|
|
||||||
function resolveDefaultCollections(
|
function resolveDefaultCollections(
|
||||||
include: boolean,
|
include: boolean,
|
||||||
workspaceDir: string,
|
workspaceDir: string,
|
||||||
@@ -283,6 +318,7 @@ export function resolveMemoryBackendConfig(params: {
|
|||||||
const command = parsedCommand?.[0] || rawCommand.split(/\s+/)[0] || "qmd";
|
const command = parsedCommand?.[0] || rawCommand.split(/\s+/)[0] || "qmd";
|
||||||
const resolved: ResolvedQmdConfig = {
|
const resolved: ResolvedQmdConfig = {
|
||||||
command,
|
command,
|
||||||
|
mcporter: resolveMcporterConfig(qmdCfg?.mcporter),
|
||||||
searchMode: resolveSearchMode(qmdCfg?.searchMode),
|
searchMode: resolveSearchMode(qmdCfg?.searchMode),
|
||||||
collections,
|
collections,
|
||||||
includeDefaultMemory,
|
includeDefaultMemory,
|
||||||
|
|||||||
@@ -185,7 +185,9 @@ export async function createEmbeddingProvider(
|
|||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
// Non-auth errors (e.g., network) are still fatal
|
// Non-auth errors (e.g., network) are still fatal
|
||||||
throw new Error(message, { cause: err });
|
const wrapped = new Error(message) as Error & { cause?: unknown };
|
||||||
|
wrapped.cause = err;
|
||||||
|
throw wrapped;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
@@ -228,7 +230,9 @@ export async function createEmbeddingProvider(
|
|||||||
};
|
};
|
||||||
}
|
}
|
||||||
// Non-auth errors are still fatal
|
// Non-auth errors are still fatal
|
||||||
throw new Error(combinedReason, { cause: fallbackErr });
|
const wrapped = new Error(combinedReason) as Error & { cause?: unknown };
|
||||||
|
wrapped.cause = fallbackErr;
|
||||||
|
throw wrapped;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
// No fallback configured - check if we should degrade to FTS-only
|
// No fallback configured - check if we should degrade to FTS-only
|
||||||
@@ -239,7 +243,9 @@ export async function createEmbeddingProvider(
|
|||||||
providerUnavailableReason: reason,
|
providerUnavailableReason: reason,
|
||||||
};
|
};
|
||||||
}
|
}
|
||||||
throw new Error(reason, { cause: primaryErr });
|
const wrapped = new Error(reason) as Error & { cause?: unknown };
|
||||||
|
wrapped.cause = primaryErr;
|
||||||
|
throw wrapped;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|||||||
@@ -148,6 +148,8 @@ describe("QmdMemoryManager", () => {
|
|||||||
afterEach(async () => {
|
afterEach(async () => {
|
||||||
vi.useRealTimers();
|
vi.useRealTimers();
|
||||||
delete process.env.OPENCLAW_STATE_DIR;
|
delete process.env.OPENCLAW_STATE_DIR;
|
||||||
|
delete (globalThis as Record<string, unknown>).__openclawMcporterDaemonStart;
|
||||||
|
delete (globalThis as Record<string, unknown>).__openclawMcporterColdStartWarned;
|
||||||
});
|
});
|
||||||
|
|
||||||
it("debounces back-to-back sync calls", async () => {
|
it("debounces back-to-back sync calls", async () => {
|
||||||
@@ -910,6 +912,170 @@ describe("QmdMemoryManager", () => {
|
|||||||
await manager.close();
|
await manager.close();
|
||||||
});
|
});
|
||||||
|
|
||||||
|
it("runs qmd searches via mcporter and warns when startDaemon=false", async () => {
|
||||||
|
cfg = {
|
||||||
|
...cfg,
|
||||||
|
memory: {
|
||||||
|
backend: "qmd",
|
||||||
|
qmd: {
|
||||||
|
includeDefaultMemory: false,
|
||||||
|
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||||
|
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||||
|
mcporter: { enabled: true, serverName: "qmd", startDaemon: false },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as OpenClawConfig;
|
||||||
|
|
||||||
|
spawnMock.mockImplementation((cmd: string, args: string[]) => {
|
||||||
|
const child = createMockChild({ autoClose: false });
|
||||||
|
if (cmd === "mcporter" && args[0] === "call") {
|
||||||
|
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
emitAndClose(child, "stdout", "[]");
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
|
||||||
|
const { manager } = await createManager();
|
||||||
|
|
||||||
|
logWarnMock.mockClear();
|
||||||
|
await expect(
|
||||||
|
manager.search("hello", { sessionKey: "agent:main:slack:dm:u123" }),
|
||||||
|
).resolves.toEqual([]);
|
||||||
|
|
||||||
|
const mcporterCalls = spawnMock.mock.calls.filter((call: unknown[]) => call[0] === "mcporter");
|
||||||
|
expect(mcporterCalls.length).toBeGreaterThan(0);
|
||||||
|
expect(mcporterCalls.some((call: unknown[]) => (call[1] as string[])[0] === "daemon")).toBe(
|
||||||
|
false,
|
||||||
|
);
|
||||||
|
expect(logWarnMock).toHaveBeenCalledWith(expect.stringContaining("cold-start"));
|
||||||
|
|
||||||
|
await manager.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("passes manager-scoped XDG env to mcporter commands", async () => {
|
||||||
|
cfg = {
|
||||||
|
...cfg,
|
||||||
|
memory: {
|
||||||
|
backend: "qmd",
|
||||||
|
qmd: {
|
||||||
|
includeDefaultMemory: false,
|
||||||
|
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||||
|
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||||
|
mcporter: { enabled: true, serverName: "qmd", startDaemon: false },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as OpenClawConfig;
|
||||||
|
|
||||||
|
spawnMock.mockImplementation((cmd: string, args: string[]) => {
|
||||||
|
const child = createMockChild({ autoClose: false });
|
||||||
|
if (cmd === "mcporter" && args[0] === "call") {
|
||||||
|
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
emitAndClose(child, "stdout", "[]");
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
|
||||||
|
const { manager } = await createManager();
|
||||||
|
await manager.search("hello", { sessionKey: "agent:main:slack:dm:u123" });
|
||||||
|
|
||||||
|
const mcporterCall = spawnMock.mock.calls.find(
|
||||||
|
(call: unknown[]) => call[0] === "mcporter" && (call[1] as string[])[0] === "call",
|
||||||
|
);
|
||||||
|
expect(mcporterCall).toBeDefined();
|
||||||
|
const spawnOpts = mcporterCall?.[2] as { env?: NodeJS.ProcessEnv } | undefined;
|
||||||
|
expect(spawnOpts?.env?.XDG_CONFIG_HOME).toContain("/agents/main/qmd/xdg-config");
|
||||||
|
expect(spawnOpts?.env?.XDG_CACHE_HOME).toContain("/agents/main/qmd/xdg-cache");
|
||||||
|
|
||||||
|
await manager.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("retries mcporter daemon start after a failure", async () => {
|
||||||
|
cfg = {
|
||||||
|
...cfg,
|
||||||
|
memory: {
|
||||||
|
backend: "qmd",
|
||||||
|
qmd: {
|
||||||
|
includeDefaultMemory: false,
|
||||||
|
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||||
|
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||||
|
mcporter: { enabled: true, serverName: "qmd", startDaemon: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as OpenClawConfig;
|
||||||
|
|
||||||
|
let daemonAttempts = 0;
|
||||||
|
spawnMock.mockImplementation((cmd: string, args: string[]) => {
|
||||||
|
const child = createMockChild({ autoClose: false });
|
||||||
|
if (cmd === "mcporter" && args[0] === "daemon") {
|
||||||
|
daemonAttempts += 1;
|
||||||
|
if (daemonAttempts === 1) {
|
||||||
|
emitAndClose(child, "stderr", "failed", 1);
|
||||||
|
} else {
|
||||||
|
emitAndClose(child, "stdout", "");
|
||||||
|
}
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
if (cmd === "mcporter" && args[0] === "call") {
|
||||||
|
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
emitAndClose(child, "stdout", "[]");
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
|
||||||
|
const { manager } = await createManager();
|
||||||
|
|
||||||
|
await manager.search("one", { sessionKey: "agent:main:slack:dm:u123" });
|
||||||
|
await manager.search("two", { sessionKey: "agent:main:slack:dm:u123" });
|
||||||
|
|
||||||
|
expect(daemonAttempts).toBe(2);
|
||||||
|
|
||||||
|
await manager.close();
|
||||||
|
});
|
||||||
|
|
||||||
|
it("starts the mcporter daemon only once when enabled", async () => {
|
||||||
|
cfg = {
|
||||||
|
...cfg,
|
||||||
|
memory: {
|
||||||
|
backend: "qmd",
|
||||||
|
qmd: {
|
||||||
|
includeDefaultMemory: false,
|
||||||
|
update: { interval: "0s", debounceMs: 60_000, onBoot: false },
|
||||||
|
paths: [{ path: workspaceDir, pattern: "**/*.md", name: "workspace" }],
|
||||||
|
mcporter: { enabled: true, serverName: "qmd", startDaemon: true },
|
||||||
|
},
|
||||||
|
},
|
||||||
|
} as OpenClawConfig;
|
||||||
|
|
||||||
|
spawnMock.mockImplementation((cmd: string, args: string[]) => {
|
||||||
|
const child = createMockChild({ autoClose: false });
|
||||||
|
if (cmd === "mcporter" && args[0] === "daemon") {
|
||||||
|
emitAndClose(child, "stdout", "");
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
if (cmd === "mcporter" && args[0] === "call") {
|
||||||
|
emitAndClose(child, "stdout", JSON.stringify({ results: [] }));
|
||||||
|
return child;
|
||||||
|
}
|
||||||
|
emitAndClose(child, "stdout", "[]");
|
||||||
|
return child;
|
||||||
|
});
|
||||||
|
|
||||||
|
const { manager } = await createManager();
|
||||||
|
|
||||||
|
await manager.search("one", { sessionKey: "agent:main:slack:dm:u123" });
|
||||||
|
await manager.search("two", { sessionKey: "agent:main:slack:dm:u123" });
|
||||||
|
|
||||||
|
const daemonStarts = spawnMock.mock.calls.filter(
|
||||||
|
(call: unknown[]) => call[0] === "mcporter" && (call[1] as string[])[0] === "daemon",
|
||||||
|
);
|
||||||
|
expect(daemonStarts).toHaveLength(1);
|
||||||
|
|
||||||
|
await manager.close();
|
||||||
|
});
|
||||||
|
|
||||||
it("fails closed when no managed collections are configured", async () => {
|
it("fails closed when no managed collections are configured", async () => {
|
||||||
cfg = {
|
cfg = {
|
||||||
...cfg,
|
...cfg,
|
||||||
|
|||||||
@@ -25,7 +25,11 @@ import type {
|
|||||||
} from "./types.js";
|
} from "./types.js";
|
||||||
|
|
||||||
type SqliteDatabase = import("node:sqlite").DatabaseSync;
|
type SqliteDatabase = import("node:sqlite").DatabaseSync;
|
||||||
import type { ResolvedMemoryBackendConfig, ResolvedQmdConfig } from "./backend-config.js";
|
import type {
|
||||||
|
ResolvedMemoryBackendConfig,
|
||||||
|
ResolvedQmdConfig,
|
||||||
|
ResolvedQmdMcporterConfig,
|
||||||
|
} from "./backend-config.js";
|
||||||
import { parseQmdQueryJson, type QmdQueryResult } from "./qmd-query-parser.js";
|
import { parseQmdQueryJson, type QmdQueryResult } from "./qmd-query-parser.js";
|
||||||
|
|
||||||
const log = createSubsystemLogger("memory");
|
const log = createSubsystemLogger("memory");
|
||||||
@@ -425,9 +429,37 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
return [];
|
return [];
|
||||||
}
|
}
|
||||||
const qmdSearchCommand = this.qmd.searchMode;
|
const qmdSearchCommand = this.qmd.searchMode;
|
||||||
|
const mcporterEnabled = this.qmd.mcporter.enabled;
|
||||||
let parsed: QmdQueryResult[];
|
let parsed: QmdQueryResult[];
|
||||||
try {
|
try {
|
||||||
if (collectionNames.length > 1) {
|
if (mcporterEnabled) {
|
||||||
|
const tool: "search" | "vector_search" | "deep_search" =
|
||||||
|
qmdSearchCommand === "search"
|
||||||
|
? "search"
|
||||||
|
: qmdSearchCommand === "vsearch"
|
||||||
|
? "vector_search"
|
||||||
|
: "deep_search";
|
||||||
|
const minScore = opts?.minScore ?? 0;
|
||||||
|
if (collectionNames.length > 1) {
|
||||||
|
parsed = await this.runMcporterAcrossCollections({
|
||||||
|
tool,
|
||||||
|
query: trimmed,
|
||||||
|
limit,
|
||||||
|
minScore,
|
||||||
|
collectionNames,
|
||||||
|
});
|
||||||
|
} else {
|
||||||
|
parsed = await this.runQmdSearchViaMcporter({
|
||||||
|
mcporter: this.qmd.mcporter,
|
||||||
|
tool,
|
||||||
|
query: trimmed,
|
||||||
|
limit,
|
||||||
|
minScore,
|
||||||
|
collection: collectionNames[0],
|
||||||
|
timeoutMs: this.qmd.limits.timeoutMs,
|
||||||
|
});
|
||||||
|
}
|
||||||
|
} else if (collectionNames.length > 1) {
|
||||||
parsed = await this.runQueryAcrossCollections(
|
parsed = await this.runQueryAcrossCollections(
|
||||||
trimmed,
|
trimmed,
|
||||||
limit,
|
limit,
|
||||||
@@ -443,7 +475,11 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
parsed = parseQmdQueryJson(result.stdout, result.stderr);
|
parsed = parseQmdQueryJson(result.stdout, result.stderr);
|
||||||
}
|
}
|
||||||
} catch (err) {
|
} catch (err) {
|
||||||
if (qmdSearchCommand !== "query" && this.isUnsupportedQmdOptionError(err)) {
|
if (
|
||||||
|
!mcporterEnabled &&
|
||||||
|
qmdSearchCommand !== "query" &&
|
||||||
|
this.isUnsupportedQmdOptionError(err)
|
||||||
|
) {
|
||||||
log.warn(
|
log.warn(
|
||||||
`qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`,
|
`qmd ${qmdSearchCommand} does not support configured flags; retrying search with qmd query`,
|
||||||
);
|
);
|
||||||
@@ -463,7 +499,8 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
throw fallbackErr instanceof Error ? fallbackErr : new Error(String(fallbackErr));
|
throw fallbackErr instanceof Error ? fallbackErr : new Error(String(fallbackErr));
|
||||||
}
|
}
|
||||||
} else {
|
} else {
|
||||||
log.warn(`qmd ${qmdSearchCommand} failed: ${String(err)}`);
|
const label = mcporterEnabled ? "mcporter/qmd" : `qmd ${qmdSearchCommand}`;
|
||||||
|
log.warn(`${label} failed: ${String(err)}`);
|
||||||
throw err instanceof Error ? err : new Error(String(err));
|
throw err instanceof Error ? err : new Error(String(err));
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
@@ -859,6 +896,169 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
});
|
});
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async ensureMcporterDaemonStarted(mcporter: ResolvedQmdMcporterConfig): Promise<void> {
|
||||||
|
if (!mcporter.enabled) {
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (!mcporter.startDaemon) {
|
||||||
|
type McporterWarnGlobal = typeof globalThis & {
|
||||||
|
__openclawMcporterColdStartWarned?: boolean;
|
||||||
|
};
|
||||||
|
const g: McporterWarnGlobal = globalThis;
|
||||||
|
if (!g.__openclawMcporterColdStartWarned) {
|
||||||
|
g.__openclawMcporterColdStartWarned = true;
|
||||||
|
log.warn(
|
||||||
|
"mcporter qmd bridge enabled but startDaemon=false; each query may cold-start QMD MCP. Consider setting memory.qmd.mcporter.startDaemon=true to keep it warm.",
|
||||||
|
);
|
||||||
|
}
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
type McporterGlobal = typeof globalThis & {
|
||||||
|
__openclawMcporterDaemonStart?: Promise<void>;
|
||||||
|
};
|
||||||
|
const g: McporterGlobal = globalThis;
|
||||||
|
if (!g.__openclawMcporterDaemonStart) {
|
||||||
|
g.__openclawMcporterDaemonStart = (async () => {
|
||||||
|
try {
|
||||||
|
await this.runMcporter(["daemon", "start"], { timeoutMs: 10_000 });
|
||||||
|
} catch (err) {
|
||||||
|
log.warn(`mcporter daemon start failed: ${String(err)}`);
|
||||||
|
// Allow future searches to retry daemon start on transient failures.
|
||||||
|
delete g.__openclawMcporterDaemonStart;
|
||||||
|
}
|
||||||
|
})();
|
||||||
|
}
|
||||||
|
await g.__openclawMcporterDaemonStart;
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runMcporter(
|
||||||
|
args: string[],
|
||||||
|
opts?: { timeoutMs?: number },
|
||||||
|
): Promise<{ stdout: string; stderr: string }> {
|
||||||
|
return await new Promise((resolve, reject) => {
|
||||||
|
const child = spawn("mcporter", args, {
|
||||||
|
// Keep mcporter and direct qmd commands on the same agent-scoped XDG state.
|
||||||
|
env: this.env,
|
||||||
|
cwd: this.workspaceDir,
|
||||||
|
});
|
||||||
|
let stdout = "";
|
||||||
|
let stderr = "";
|
||||||
|
let stdoutTruncated = false;
|
||||||
|
let stderrTruncated = false;
|
||||||
|
const timer = opts?.timeoutMs
|
||||||
|
? setTimeout(() => {
|
||||||
|
child.kill("SIGKILL");
|
||||||
|
reject(new Error(`mcporter ${args.join(" ")} timed out after ${opts.timeoutMs}ms`));
|
||||||
|
}, opts.timeoutMs)
|
||||||
|
: null;
|
||||||
|
child.stdout.on("data", (data) => {
|
||||||
|
const next = appendOutputWithCap(stdout, data.toString("utf8"), this.maxQmdOutputChars);
|
||||||
|
stdout = next.text;
|
||||||
|
stdoutTruncated = stdoutTruncated || next.truncated;
|
||||||
|
});
|
||||||
|
child.stderr.on("data", (data) => {
|
||||||
|
const next = appendOutputWithCap(stderr, data.toString("utf8"), this.maxQmdOutputChars);
|
||||||
|
stderr = next.text;
|
||||||
|
stderrTruncated = stderrTruncated || next.truncated;
|
||||||
|
});
|
||||||
|
child.on("error", (err) => {
|
||||||
|
if (timer) {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
reject(err);
|
||||||
|
});
|
||||||
|
child.on("close", (code) => {
|
||||||
|
if (timer) {
|
||||||
|
clearTimeout(timer);
|
||||||
|
}
|
||||||
|
if (stdoutTruncated || stderrTruncated) {
|
||||||
|
reject(
|
||||||
|
new Error(
|
||||||
|
`mcporter ${args.join(" ")} produced too much output (limit ${this.maxQmdOutputChars} chars)`,
|
||||||
|
),
|
||||||
|
);
|
||||||
|
return;
|
||||||
|
}
|
||||||
|
if (code === 0) {
|
||||||
|
resolve({ stdout, stderr });
|
||||||
|
} else {
|
||||||
|
reject(
|
||||||
|
new Error(`mcporter ${args.join(" ")} failed (code ${code}): ${stderr || stdout}`),
|
||||||
|
);
|
||||||
|
}
|
||||||
|
});
|
||||||
|
});
|
||||||
|
}
|
||||||
|
|
||||||
|
private async runQmdSearchViaMcporter(params: {
|
||||||
|
mcporter: ResolvedQmdMcporterConfig;
|
||||||
|
tool: "search" | "vector_search" | "deep_search";
|
||||||
|
query: string;
|
||||||
|
limit: number;
|
||||||
|
minScore: number;
|
||||||
|
collection?: string;
|
||||||
|
timeoutMs: number;
|
||||||
|
}): Promise<QmdQueryResult[]> {
|
||||||
|
await this.ensureMcporterDaemonStarted(params.mcporter);
|
||||||
|
|
||||||
|
const selector = `${params.mcporter.serverName}.${params.tool}`;
|
||||||
|
const callArgs: Record<string, unknown> = {
|
||||||
|
query: params.query,
|
||||||
|
limit: params.limit,
|
||||||
|
minScore: params.minScore,
|
||||||
|
};
|
||||||
|
if (params.collection) {
|
||||||
|
callArgs.collection = params.collection;
|
||||||
|
}
|
||||||
|
|
||||||
|
const result = await this.runMcporter(
|
||||||
|
[
|
||||||
|
"call",
|
||||||
|
selector,
|
||||||
|
"--args",
|
||||||
|
JSON.stringify(callArgs),
|
||||||
|
"--output",
|
||||||
|
"json",
|
||||||
|
"--timeout",
|
||||||
|
String(Math.max(0, params.timeoutMs)),
|
||||||
|
],
|
||||||
|
{ timeoutMs: Math.max(params.timeoutMs + 2_000, 5_000) },
|
||||||
|
);
|
||||||
|
|
||||||
|
const parsedUnknown: unknown = JSON.parse(result.stdout);
|
||||||
|
const isRecord = (value: unknown): value is Record<string, unknown> =>
|
||||||
|
typeof value === "object" && value !== null && !Array.isArray(value);
|
||||||
|
|
||||||
|
const structured =
|
||||||
|
isRecord(parsedUnknown) && isRecord(parsedUnknown.structuredContent)
|
||||||
|
? parsedUnknown.structuredContent
|
||||||
|
: parsedUnknown;
|
||||||
|
|
||||||
|
const results: unknown[] =
|
||||||
|
isRecord(structured) && Array.isArray(structured.results)
|
||||||
|
? (structured.results as unknown[])
|
||||||
|
: Array.isArray(structured)
|
||||||
|
? structured
|
||||||
|
: [];
|
||||||
|
|
||||||
|
const out: QmdQueryResult[] = [];
|
||||||
|
for (const item of results) {
|
||||||
|
if (!isRecord(item)) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const docidRaw = item.docid;
|
||||||
|
const docid = typeof docidRaw === "string" ? docidRaw.replace(/^#/, "").trim() : "";
|
||||||
|
if (!docid) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const scoreRaw = item.score;
|
||||||
|
const score = typeof scoreRaw === "number" ? scoreRaw : Number(scoreRaw);
|
||||||
|
const snippet = typeof item.snippet === "string" ? item.snippet : "";
|
||||||
|
out.push({ docid, score: Number.isFinite(score) ? score : 0, snippet });
|
||||||
|
}
|
||||||
|
return out;
|
||||||
|
}
|
||||||
|
|
||||||
private async readPartialText(
|
private async readPartialText(
|
||||||
absPath: string,
|
absPath: string,
|
||||||
from?: number,
|
from?: number,
|
||||||
@@ -1407,6 +1607,39 @@ export class QmdMemoryManager implements MemorySearchManager {
|
|||||||
return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
||||||
}
|
}
|
||||||
|
|
||||||
|
private async runMcporterAcrossCollections(params: {
|
||||||
|
tool: "search" | "vector_search" | "deep_search";
|
||||||
|
query: string;
|
||||||
|
limit: number;
|
||||||
|
minScore: number;
|
||||||
|
collectionNames: string[];
|
||||||
|
}): Promise<QmdQueryResult[]> {
|
||||||
|
const bestByDocId = new Map<string, QmdQueryResult>();
|
||||||
|
for (const collectionName of params.collectionNames) {
|
||||||
|
const parsed = await this.runQmdSearchViaMcporter({
|
||||||
|
mcporter: this.qmd.mcporter,
|
||||||
|
tool: params.tool,
|
||||||
|
query: params.query,
|
||||||
|
limit: params.limit,
|
||||||
|
minScore: params.minScore,
|
||||||
|
collection: collectionName,
|
||||||
|
timeoutMs: this.qmd.limits.timeoutMs,
|
||||||
|
});
|
||||||
|
for (const entry of parsed) {
|
||||||
|
if (typeof entry.docid !== "string" || !entry.docid.trim()) {
|
||||||
|
continue;
|
||||||
|
}
|
||||||
|
const prev = bestByDocId.get(entry.docid);
|
||||||
|
const prevScore = typeof prev?.score === "number" ? prev.score : Number.NEGATIVE_INFINITY;
|
||||||
|
const nextScore = typeof entry.score === "number" ? entry.score : Number.NEGATIVE_INFINITY;
|
||||||
|
if (!prev || nextScore > prevScore) {
|
||||||
|
bestByDocId.set(entry.docid, entry);
|
||||||
|
}
|
||||||
|
}
|
||||||
|
}
|
||||||
|
return [...bestByDocId.values()].toSorted((a, b) => (b.score ?? 0) - (a.score ?? 0));
|
||||||
|
}
|
||||||
|
|
||||||
private listManagedCollectionNames(): string[] {
|
private listManagedCollectionNames(): string[] {
|
||||||
const seen = new Set<string>();
|
const seen = new Set<string>();
|
||||||
const names: string[] = [];
|
const names: string[] = [];
|
||||||
|
|||||||
Reference in New Issue
Block a user