aaronrene / knowtation public
docs-import-propose.mjs
165 lines 6.1 KB
Raw
sha256:700fafdd1afa490919f9515d660ca6e75456bcd5bb67513abcd8757a634c01f6 docs: record AIP-b SD-21 land (KN #308) Human 9 days ago
1 /**
2 * Review-before-write proposal builder for live document connectors.
3 *
4 * This module never writes notes. It creates proposal records after enforcing
5 * frozen file-count and byte caps and deduplicating canonical and pending work.
6 */
7
8 import fs from 'fs';
9 import path from 'path';
10 import { createProposal as createStoredProposal } from '../../hub/proposals-store.mjs';
11 import { noteStateIdFromParts } from '../note-state-id.mjs';
12 import { listMarkdownFiles, readNote } from '../vault.mjs';
13 import { safeId } from './google-drive-normalizer.mjs';
14
15 export const MAX_DOCS_IMPORT_FILES = 20;
16 export const MAX_DOCS_IMPORT_FILE_BYTES = 25_000_000;
17 export const MAX_DOCS_IMPORT_BATCH_BYTES = 80_000_000;
18 export const DOCS_SYNC_REVIEW_QUEUE = 'docs-sync';
19
20 function badRequest(message) {
21 const error = new TypeError(message);
22 error.code = 'BAD_REQUEST';
23 return error;
24 }
25
26 function defaultLoadProposals(dataDir) {
27 const filePath = path.join(dataDir, 'hub_proposals.json');
28 if (!fs.existsSync(filePath)) return [];
29 try {
30 const parsed = JSON.parse(fs.readFileSync(filePath, 'utf8'));
31 return Array.isArray(parsed) ? parsed : [];
32 } catch {
33 return [];
34 }
35 }
36
37 function isPendingProposal(proposal, vaultId, source, sourceId) {
38 if (!proposal || !['proposed', 'approved'].includes(proposal.status)) return false;
39 if ((proposal.vault_id ?? 'default') !== vaultId) return false;
40 return proposal.frontmatter?.source === source
41 && proposal.frontmatter?.source_id === sourceId;
42 }
43
44 function findExistingNote(vaultPath, source, sourceId, listFilesFn, readNoteFn) {
45 for (const relativePath of listFilesFn(vaultPath)) {
46 let note;
47 try {
48 note = readNoteFn(vaultPath, relativePath);
49 } catch {
50 continue;
51 }
52 if (note.frontmatter?.source === source && note.frontmatter?.source_id === sourceId) {
53 return note;
54 }
55 }
56 return null;
57 }
58
59 /**
60 * Create review proposals for already-fetched provider documents.
61 *
62 * @param {{
63 * dataDir: string,
64 * vaultPath: string,
65 * vaultId: string,
66 * connectorId: string,
67 * provider: 'google-drive'|'notion',
68 * items: Array<{ source_id?: string, file_id?: string, page_id?: string, name?: string, markdown: string, size?: number }>,
69 * now?: number|string|Date,
70 * createProposalFn?: typeof createStoredProposal,
71 * loadProposalsFn?: (dataDir: string) => object[],
72 * listMarkdownFilesFn?: typeof listMarkdownFiles,
73 * readNoteFn?: typeof readNote,
74 * }} input
75 * @returns {{ proposed: number, skipped: number, proposal_ids: string[], skip_details: Array<{ source_id: string, reason: string }> }}
76 */
77 export function proposeDocsImports(input) {
78 const items = Array.isArray(input?.items) ? input.items : null;
79 if (!items || items.length < 1 || items.length > MAX_DOCS_IMPORT_FILES) {
80 throw badRequest('file_ids must contain between 1 and 20 items');
81 }
82 if (!['google-drive', 'notion'].includes(input.provider)) throw badRequest('provider denied');
83
84 let batchBytes = 0;
85 for (const item of items) {
86 const size = Number.isFinite(item.size)
87 ? Number(item.size)
88 : Buffer.byteLength(typeof item.markdown === 'string' ? item.markdown : '', 'utf8');
89 if (size < 0) throw badRequest('invalid file size');
90 batchBytes += size;
91 }
92 if (batchBytes > MAX_DOCS_IMPORT_BATCH_BYTES) throw badRequest('import batch exceeds byte cap');
93
94 const source = input.provider;
95 const vaultId = typeof input.vaultId === 'string' && input.vaultId ? input.vaultId : 'default';
96 const createFn = input.createProposalFn ?? createStoredProposal;
97 const loadFn = input.loadProposalsFn ?? defaultLoadProposals;
98 const listFilesFn = input.listMarkdownFilesFn ?? listMarkdownFiles;
99 const readNoteFn = input.readNoteFn ?? readNote;
100 const pending = loadFn(input.dataDir);
101 const importedAt = new Date(input.now ?? Date.now()).toISOString();
102 const proposalIds = [];
103 const skipDetails = [];
104
105 for (const item of items) {
106 const sourceId = item.source_id ?? item.file_id ?? item.page_id;
107 if (typeof sourceId !== 'string' || !sourceId) throw badRequest('invalid source id');
108 const itemSize = Number.isFinite(item.size)
109 ? Number(item.size)
110 : Buffer.byteLength(typeof item.markdown === 'string' ? item.markdown : '', 'utf8');
111 if (itemSize > MAX_DOCS_IMPORT_FILE_BYTES) {
112 skipDetails.push({ source_id: sourceId, reason: 'too_large' });
113 continue;
114 }
115 if (typeof item.markdown !== 'string' || !item.markdown.trim()) {
116 skipDetails.push({ source_id: sourceId, reason: 'empty_extract' });
117 continue;
118 }
119 if (pending.some((proposal) => isPendingProposal(proposal, vaultId, source, sourceId))) {
120 skipDetails.push({ source_id: sourceId, reason: 'already_pending' });
121 continue;
122 }
123 const pathId = safeId(sourceId);
124 if (!pathId) throw badRequest('invalid source id');
125 const existing = findExistingNote(input.vaultPath, source, sourceId, listFilesFn, readNoteFn);
126 const proposalPath = existing?.path ?? `imports/${source}/${pathId}.md`;
127 const displayName = typeof item.name === 'string' && item.name.trim()
128 ? item.name.trim()
129 : sourceId;
130 const prefix = 'docs-sync import: ';
131 const intent = prefix + displayName.slice(0, 128 - prefix.length);
132 const frontmatter = {
133 source,
134 source_id: sourceId,
135 connector_id: input.connectorId,
136 imported_at: importedAt,
137 };
138 const proposal = createFn(input.dataDir, {
139 path: proposalPath,
140 body: item.markdown,
141 frontmatter,
142 intent,
143 vault_id: vaultId,
144 source: 'import',
145 review_queue: DOCS_SYNC_REVIEW_QUEUE,
146 ...(existing
147 ? { base_state_id: noteStateIdFromParts(existing.frontmatter ?? {}, existing.body ?? '') }
148 : {}),
149 });
150 if (!proposal || typeof proposal.proposal_id !== 'string') {
151 throw new Error('Proposal store did not return a proposal id');
152 }
153 proposalIds.push(proposal.proposal_id);
154 pending.push(proposal);
155 }
156
157 return {
158 proposed: proposalIds.length,
159 skipped: skipDetails.length,
160 proposal_ids: proposalIds,
161 skip_details: skipDetails,
162 };
163 }
164
165 export const createDocsImportProposals = proposeDocsImports;
File History 1 commit
sha256:700fafdd1afa490919f9515d660ca6e75456bcd5bb67513abcd8757a634c01f6 docs: record AIP-b SD-21 land (KN #308) Human 9 days ago