Fix intermittent scraper 504s and poisoned cache: single instance, 504 on timeout, 1h TTL

This commit is contained in:
2026-10-22 01:07:26 -04:00
parent 670707a4c8
commit 825f64e62a
4 changed files with 165 additions and 16 deletions
+8 -1
View File
@@ -3,5 +3,12 @@ module.exports = [{
name: 'prerender-app',
cwd: '/home/node',
exec_mode: 'cluster',
instances: process.env.PHANTOM_WORKERS || 3, // Adjust based on your CPU cores
// IMPORTANT: keep instances at 1. Every prerender instance spawns its own
// Chrome bound to debugging port 9222 (see server.js chromeFlags); with more
// than one instance they all fight over that port, all connect to whichever
// Chrome bound it first, and when that Chrome restarts every worker loses its
// websocket mid-request -> intermittent 504s for real crawlers (facebookexternalhit
// etc). One instance already serves N concurrent renders, one Chrome tab per
// request. See README "Reliability design".
instances: 1,
}];