Bulk rebuild candidate indexes and retain concurrent config edits

Author: Forge

CHANGELOG.md

@@ -17,7 +17,7 @@ and adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
 
 ### Extension preview
 
-- Reconcile global semantic-extension links after sync or explicit file indexing with an atomic candidate rebuild. Failed passes preserve the working graph. Reuse bounded core parse results on warm project instances while rerunning all framework hooks, global resolution and semantic passes; cold or disabled reuse retains full parsing. Reject observed source changes before commit and reclaim interrupted graph-only candidates. Rebuild text search in bulk inside the atomic replacement transaction.
+- Reconcile global semantic-extension links after sync or explicit file indexing with an atomic candidate rebuild. Failed passes preserve the working graph. Reuse bounded core parse results on warm project instances while rerunning all framework hooks, global resolution and semantic passes; cold or disabled reuse retains full parsing. Reject observed source changes before commit and reclaim interrupted graph-only candidates. Rebuild text search and built-in secondary indexes in bulk inside the atomic replacement transaction.
 
 - Prepare immutable Drupal extension 0.1.1: ignore unrelated static receivers and commented/quoted declarations, and resolve both explicit Symfony event argument orders. Preserve the reviewed 0.1.0 artifact and policy.
 - Keep installed and selected marketplace versions distinct while compatibility metadata refreshes; pending requests no longer display an incompatible catalog version as installed.

__tests__/extension-sync.test.ts

@@ -48,13 +48,13 @@ it('reuses core parsing while rerunning global hooks, rejects source races, isol
   const author = path.join(root, 'author'); createExtensionProject(author, 'global-fixture');
   const manifestPath = path.join(author, 'package.json'), manifest = JSON.parse(fs.readFileSync(manifestPath, 'utf8'));
   manifest.codegraph.capabilities = ['frameworks', 'synthPasses']; fs.writeFileSync(manifestPath, JSON.stringify(manifest));
-  fs.writeFileSync(path.join(author, 'index.cjs'), `module.exports = ({projectRoot}) => ({
+  fs.writeFileSync(path.join(author, 'index.cjs'), `module.exports = ({projectRoot,options}) => ({
     frameworks:[{name:'global-framework',languages:['python'],detect:()=>true,resolve:()=>null,
       extract(file){ if(file!=='handlers.py')return {nodes:[],references:[]};
         const name=require('fs').readFileSync(require('path').join(projectRoot,'binding.py'),'utf8').trim();
         return {nodes:[{id:'plugin:global-fixture:route',kind:'route',name,qualifiedName:name,filePath:file,language:'python',startLine:1,endLine:1,startColumn:0,endColumn:1,metadata:{}}],references:[]}; }}],
     synthPasses:[{name:'bindings',languages:['python'],run(ctx){
-      const name=(ctx.readFile('binding.py')||'').trim(); if(name==='FAIL')throw Error('real pass failed');
+      const name=options.target||(ctx.readFile('binding.py')||'').trim(); if(name==='FAIL')throw Error('real pass failed');
       const from=ctx.getNodesByName('dispatch').filter(n=>n.kind==='function'),to=ctx.getNodesByName(name).filter(n=>n.kind==='function');if(from.length!==1||to.length!==1)return [];
       return [{source:from[0].id,target:to[0].id,kind:'calls',line:1,metadata:{label:'Global binding'}}]; }}]
   });`);
@@ -91,6 +91,18 @@ it('reuses core parsing while rerunning global hooks, rejects source races, isol
     expect(edges(g)).toEqual([['dispatch','cancel']]);
     fs.writeFileSync(binding,'receipt'); await g.sync(); expect(edges(g)).toEqual([['dispatch','receipt']]);
     expect(edges(graphs[1])).toEqual([['dispatch','cancel']]);
+    // A config edit after the final source observation must remain dirty. The
+    // committed stamp must describe the evaluated options, not the later file.
+    const configFile = path.join(project, 'codegraph.json'), config = JSON.parse(fs.readFileSync(configFile, 'utf8'));
+    const editDuringCopy = (event: any) => { if (event.projectRoot === project && event.phase === 'copy_started') {
+      config.plugins[0].options = { target: 'cancel' }; fs.writeFileSync(configFile, JSON.stringify(config));
+    }};
+    diagnostics.subscribe(editDuringCopy);
+    try { await g.indexFiles(['handlers.py']); } finally { diagnostics.unsubscribe(editDuringCopy); }
+    expect(edges(g)).toEqual([['dispatch','receipt']]);
+    expect(JSON.parse(fs.readFileSync(configFile,'utf8')).plugins[0].options.target).toBe('cancel');
+    await g.sync(); expect(edges(g)).toEqual([['dispatch','cancel']]);
+    config.plugins[0].options = {}; fs.writeFileSync(configFile,JSON.stringify(config)); await g.sync();
     const before = JSON.stringify([edges(g),g.getNodesByKind('route')]);
     const fresh = await CodeGraph.open(project); graphs.push(fresh); await fresh.refreshPluginIndex();
     expect(JSON.stringify([edges(fresh),fresh.getNodesByKind('route')])).toBe(before);

docs/design/semantic-update-reuse.md

@@ -30,19 +30,21 @@ Before/after source stamps cover the scanned indexed files plus root config and
 ignore rules. An observed change aborts before commit and preserves the working
 graph. This does not lock the filesystem: arbitrary external plugin I/O or a
 write after the final observation is not a certified snapshot. Retry after edits
-settle. Cache entries are content-keyed, so a failed candidate may safely retain
+settle. The committed plugin fingerprint is frozen at registry load, so a config
+edit during the subsequent SQLite copy stays dirty for the next sync instead of
+incorrectly labelling an old graph as matching new options. Cache entries are content-keyed, so a failed candidate may safely retain
 unchanged core parses but never graph or plugin output.
 
 Graph-only candidate records contain a checksummed UUID and diagnostic PID.
 The project SQLite coordinator is authoritative, independent of PID reuse.
 Fresh opens reclaim the exact orphaned stage after process death; an invalid
 record blocks recovery without deleting files. During a live graph-only update,
 readers can use the old committed graph. The existing SQLite graph transaction
-still determines old/new visibility. FTS triggers are suspended and rebuilt
+still determines old/new visibility. Built-in non-unique graph secondary indexes and FTS triggers are suspended and rebuilt
 inside that same transaction using the existing bulk-load API, avoiding per-row
-FTS work during replacement. A kill rolls back the trigger schema and FTS data
+index maintenance during replacement. A kill rolls back the secondary/trigger schema and FTS data
 with the graph; readers never see a committed missing-trigger window.
-`CODEGRAPH_NO_SEMANTIC_BULK_FTS=1` retains the previous trigger path. A commit marker invalidates long-lived
+`CODEGRAPH_NO_SEMANTIC_BULK=1` retains the previous trigger path. A commit marker invalidates long-lived
 reader caches. Managed config/trust transactions retain their exclusive guard.
 
 Measurements and correctness/recovery coverage will be linked after validation.

scripts/validation/semantic-update-recovery.cjs

@@ -5,6 +5,7 @@ const {CodeGraph,createExtensionProject,packExtension}=require('../../dist');con
 const {DatabaseSync}=require('node:sqlite');
 Object.assign(process.env,{CODEGRAPH_TELEMETRY:'0',CODEGRAPH_PARSE_WORKERS:'2',CODEGRAPH_RESOLVE_WORKERS:'2',CODEGRAPH_PARALLEL_RESOLVE_MIN:'0'});
 const phases=['candidate_ready','before_commit','copy_started','in_commit','committed'];
+function schema(root){const db=new DatabaseSync(path.join(root,'.codegraph/codegraph.db'),{readOnly:true});try{return db.prepare("SELECT name,sql FROM sqlite_master WHERE type IN ('index','trigger') AND sql IS NOT NULL ORDER BY name").all().map(r=>[r.name,r.sql])}finally{db.close()}}
 function ftsAdded(root){const db=new DatabaseSync(path.join(root,'.codegraph/codegraph.db'),{readOnly:true});try{return db.prepare("SELECT count(*) n FROM nodes_fts WHERE nodes_fts MATCH 'added_function'").get().n}finally{db.close()}}
 function snapshot(root){const db=new DatabaseSync(path.join(root,'.codegraph/codegraph.db'),{readOnly:true});try{return db.prepare("select a.name source,b.name target from edges e join nodes a on a.id=e.source join nodes b on b.id=e.target where e.metadata like '%\"synthesizedBy\":\"kill-fixture\"%' order by a.name,b.name").all().map(r=>[r.source,r.target]);}finally{db.close()}}
 if(process.argv[2]==='--child'){
@@ -25,15 +26,15 @@ if(process.argv[2]==='--child'){
  try{
  const author=path.join(lab,'author');createExtensionProject(author,'kill-fixture');const manifest=JSON.parse(fs.readFileSync(path.join(author,'package.json'),'utf8'));manifest.codegraph.capabilities=['synthPasses'];fs.writeFileSync(path.join(author,'package.json'),JSON.stringify(manifest));
  fs.writeFileSync(path.join(author,'index.cjs'),`module.exports=()=>({synthPasses:[{name:'binding',languages:['python'],run(ctx){const name=(ctx.readFile('binding.py')||'').trim();const a=ctx.getNodesByName('dispatch'),b=ctx.getNodesByName(name);if(a.length!==1||b.length!==1)return [];return [{source:a[0].id,target:b[0].id,kind:'calls',metadata:{label:'fixture'}}]}}]});`);const bytes=packExtension(author);
- for(const phase of phases){const root=path.join(lab,phase);fs.mkdirSync(root);fs.writeFileSync(path.join(root,'handlers.py'),'def dispatch():\n    missing()\ndef receipt():\n    pass\ndef cancel():\n    pass\n');fs.writeFileSync(path.join(root,'binding.py'),'receipt');await new ExtensionManager(root).install({bytes});const config=fs.readFileSync(path.join(root,'codegraph.json'),'utf8');
+ for(const phase of phases){const root=path.join(lab,phase);fs.mkdirSync(root);fs.writeFileSync(path.join(root,'handlers.py'),'def dispatch():\n    missing()\ndef receipt():\n    pass\ndef cancel():\n    pass\n');fs.writeFileSync(path.join(root,'binding.py'),'receipt');await new ExtensionManager(root).install({bytes});const custom=new DatabaseSync(path.join(root,'.codegraph/codegraph.db'));custom.exec('CREATE INDEX user_fixture_name ON nodes(name)');custom.close();const originalSchema=schema(root);const config=fs.readFileSync(path.join(root,'codegraph.json'),'utf8');
  const reader=await CodeGraph.open(root);assert.deepEqual(snapshot(root),[['dispatch','receipt']]);reader.getNodesByKind('function');
  const c=child(root,'update',phase);let timer;try{await Promise.race([c.reached,c.done.then(r=>{throw Error('child ended '+r.stderr)}),new Promise((_,r)=>timer=setTimeout(()=>r(Error('timeout '+phase)),90000))])}finally{clearTimeout(timer)}
- const expected=phase==='committed'?'cancel':'receipt';assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.ok(reader.getStats().nodeCount>0);assert.equal(ftsAdded(root),phase==='committed'?1:0);
+ const expected=phase==='committed'?'cancel':'receipt';assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.ok(reader.getStats().nodeCount>0);assert.equal(ftsAdded(root),phase==='committed'?1:0);assert.deepEqual(schema(root),originalSchema);
  const concurrent=await child(root,'retry').done;assert.notEqual(concurrent.exit,0);assert.match(concurrent.stderr,/operation is active/);
  assert.equal(c.p.kill('SIGKILL'),true);const killed=await c.done;assert.ok(killed.signal==='SIGKILL'||(process.platform==='win32'&&killed.exit!==0));
- fs.writeFileSync(path.join(root,'keep.txt'),'unrelated user file');const inspect=await child(root,'inspect').done;assert.equal(inspect.exit,0,inspect.stderr);assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.equal(ftsAdded(root),phase==='committed'?1:0);
+ fs.writeFileSync(path.join(root,'keep.txt'),'unrelated user file');const inspect=await child(root,'inspect').done;assert.equal(inspect.exit,0,inspect.stderr);assert.deepEqual(snapshot(root),[['dispatch',expected]]);assert.equal(ftsAdded(root),phase==='committed'?1:0);assert.deepEqual(schema(root),originalSchema);
  assert.equal(fs.existsSync(path.join(root,'.codegraph/plugins/candidate.json')),false);assert.deepEqual(fs.readdirSync(path.join(root,'.codegraph')).filter(f=>f.startsWith('extension-stage-')),[]);
- assert.equal((await child(root,'inspect').done).exit,0);assert.equal((await child(root,'retry').done).exit,0);assert.deepEqual(snapshot(root),[['dispatch','cancel']]);assert.equal(ftsAdded(root),1);
+ assert.equal((await child(root,'inspect').done).exit,0);assert.equal((await child(root,'retry').done).exit,0);assert.deepEqual(snapshot(root),[['dispatch','cancel']]);assert.equal(ftsAdded(root),1);assert.deepEqual(schema(root),originalSchema);
  const funcs=reader.getNodesByKind('function');const names=new Map(funcs.map(n=>[n.id,n.name]));assert.deepEqual(reader.getOutgoingEdgesFrom(funcs.map(n=>n.id)).filter(e=>e.metadata?.synthesizedBy==='kill-fixture').map(e=>[names.get(e.source),names.get(e.target)]),[['dispatch','cancel']]);reader.close();
  assert.equal(fs.readFileSync(path.join(root,'codegraph.json'),'utf8'),config);assert.equal(fs.readFileSync(path.join(root,'keep.txt'),'utf8'),'unrelated user file');rows.push({phase,passed:true,edgesAfterKill:expected,concurrentExcluded:true,repeatedRecovery:true,readerFresh:true});save();console.log('PASS',phase);
  }

src/db/index.ts

@@ -303,6 +303,20 @@ export class DatabaseConnection {
     await this.endBulkEdgeLoad();
   }
 
+  /** @internal Synchronous restoration for a whole-graph replacement transaction.
+   * The caller deletes ALL graph rows, so no mid-load queries need secondary
+   * indexes. Unlike ordinary async indexing, no yield may escape the enclosing
+   * SQLite transaction. Primary/unique constraints and user indexes stay intact.
+   */
+  endAtomicBulkParseLoad(): void {
+    const schema = fs.readFileSync(path.join(__dirname, 'schema.sql'), 'utf-8');
+    for (const index of [...DatabaseConnection.BULK_PARSE_INDEX_NAMES, ...DatabaseConnection.BULK_EDGE_INDEX_NAMES]) {
+      const statement = schema.match(new RegExp(`CREATE INDEX IF NOT EXISTS ${index}\\b[^;]*;`));
+      if (!statement) throw new Error(`schema.sql: index ${index} not found for atomic bulk recreation`);
+      this.db.exec(statement[0]);
+    }
+  }
+
   /**
    * unresolved_refs secondary indexes NOT read by the batched resolution
    * loop. The loop pages pending refs by keyset (`status='pending' AND id>?`

src/index.ts

@@ -567,7 +567,12 @@ export class CodeGraph {
       let candidate: CodeGraph | undefined;
       let attached = false;
       try {
+        const configFile = path.join(this.projectRoot, 'codegraph.json');
+        const readConfig = () => fs.existsSync(configFile) ? fs.readFileSync(configFile, 'utf8') : null;
+        const configAtLoad = readConfig();
         const plugins = await loadPlugins(this.projectRoot);
+        const activatedStamp = this.extensionStamp(plugins);
+        if (readConfig() !== configAtLoad) throw new Error('Project configuration changed while loading extensions; previous graph retained. Retry the update.');
         if (plugins.diagnostics.some(d => d.state !== 'loaded')) throw new Error(plugins.diagnostics.map(d => `${d.id}: ${d.message ?? d.state}`).join('; '));
         const inputs = plugins.synthPasses.length ? this.semanticInputStamp() : undefined;
         if (!options.extensionTransactionId) { beginGraphCandidate(this.projectRoot, candidateId); recorded = true; }
@@ -597,16 +602,18 @@ export class CodeGraph {
           // Candidate replacement is one bulk load. Keep the FTS trigger window
           // INSIDE this SQLite transaction so a kill rolls back graph and FTS
           // schema together; readers keep the prior committed index.
-          const bulkFts = process.env.CODEGRAPH_NO_SEMANTIC_BULK_FTS !== '1';
-          if (bulkFts) this.db.beginBulkNodeLoad();
+          const bulkGraphIndexes = process.env.CODEGRAPH_NO_SEMANTIC_BULK !== '1';
+          if (bulkGraphIndexes) { this.db.beginBulkNodeLoad(); this.db.beginBulkParseLoad(); }
           channel('codegraph.semantic.update').publish({ projectRoot: this.projectRoot, phase: 'copy_started' });
           this.queries.clear();
           for (const table of ['files', 'nodes', 'edges', 'unresolved_refs']) {
             connection.exec(`INSERT INTO main.${table} SELECT * FROM extension_candidate.${table}`);
           }
-          if (bulkFts) this.db.endBulkNodeLoad();
+          if (bulkGraphIndexes) { this.db.endAtomicBulkParseLoad(); this.db.endBulkNodeLoad(); }
           this.queries.setMetadata('plugins_last_run', JSON.stringify(plugins.diagnostics));
-          this.queries.setMetadata('indexed_with_plugins', this.extensionStamp(plugins));
+          // Freeze the evaluated configuration. An edit during the SQLite copy
+          // must stay dirty for the next sync, never stamp old output as current.
+          this.queries.setMetadata('indexed_with_plugins', activatedStamp);
           this.queries.setMetadata('indexed_with_version', CodeGraphPackageVersion);
           this.queries.setMetadata('indexed_with_extraction_version', String(EXTRACTION_VERSION));
           this.queries.setMetadata('index_state', 'complete');