Complete overhaul - changed to pm2-runtime and the excellent prerender-plugin-fscache. Working excellently!

.dockerignore

@@ -0,0 +1 @@
+.env

Dockerfile

@@ -17,15 +17,15 @@ ENV CHROME_FLAGS="--headless --disable-gpu --no-sandbox --disable-dev-shm-usage
 
 WORKDIR /home/node
 
+USER root
+RUN mkdir -p /home/node/prerender-cache && chown -R node:node /home/node/prerender-cache
 RUN apt-get update && apt-get install -y \
     chromium \
     --no-install-recommends \
     && rm -rf /var/lib/apt/lists/*
 
-# Already has git
-# RUN apt-get update -y
-RUN yarn add prerender prerender-memory-cache pm2 winston nodemailer
-
 USER node:node
+RUN yarn add prerender pm2 prerender-plugin-fscache
 
-ENTRYPOINT ["node", "server.js"]
+
+ENTRYPOINT ["pm2-runtime", "start", "ecosystem.config.js"]

PM2Manager

@@ -1,340 +0,0 @@
-// To use this enhanced version, you'll need to:
-
-// Install additional dependencies:
-// npm install winston nodemailer node-fetch
-
-// Set up environment variables:
-// export NOTIFICATION_EMAIL=alerts@yourdomain.com
-// export ADMIN_EMAIL=admin@yourdomain.com
-// export SMTP_HOST=smtp.yourdomain.com
-// export SMTP_PORT=587
-// export SMTP_USER=your-user
-// export SMTP_PASS=your-password
-
-
-const { spawn, exec } = require('child_process');
-const pm2 = require('pm2');
-const fs = require('fs');
-const path = require('path');
-const winston = require('winston');
-const nodemailer = require('nodemailer');
-
-// Configuration file
-const config = {
-    app: {
-        name: 'server',
-        script: 'server.js',
-        instances: 4,
-        max_memory_restart: '300M',
-        max_restarts: 10,
-        exp_backoff_restart_delay: 100
-    },
-    monitoring: {
-        healthCheckInterval: 30000,
-        healthCheckEndpoint: 'http://localhost:3000/health',
-        metricsInterval: 60000,
-        maxRestartAttempts: 5,
-        criticalErrors: ['page timed out', 'parse html timed out', 'Timed out waiting for']
-    },
-    notifications: {
-        email: {
-            enabled: true,
-            from: process.env.NOTIFICATION_EMAIL,
-            to: process.env.ADMIN_EMAIL,
-            smtp: {
-                host: process.env.SMTP_HOST,
-                port: process.env.SMTP_PORT,
-                auth: {
-                    user: process.env.SMTP_USER,
-                    pass: process.env.SMTP_PASS
-                }
-            }
-        }
-    },
-    logging: {
-        dir: 'logs',
-        maxSize: '10m',
-        maxFiles: '7d'
-    }
-};
-
-// Save config to file
-fs.writeFileSync('pm2-config.json', JSON.stringify(config, null, 2));
-
-// Initialize logger
-const logger = winston.createLogger({
-    level: 'info',
-    format: winston.format.combine(
-        winston.format.timestamp(),
-        winston.format.json()
-    ),
-    transports: [
-        new winston.transports.Console(),
-        new winston.transports.File({
-            filename: path.join(config.logging.dir, 'error.log'),
-            level: 'error',
-            maxsize: 5242880, // 5MB
-            maxFiles: 5
-        }),
-        new winston.transports.File({
-            filename: path.join(config.logging.dir, 'combined.log'),
-            maxsize: 5242880,
-            maxFiles: 5
-        })
-    ]
-});
-
-// Metrics tracking
-class MetricsTracker {
-    constructor() {
-        this.metrics = {
-            startTime: Date.now(),
-            restarts: 0,
-            lastRestart: null,
-            uptimePercentage: 100,
-            memoryUsage: [],
-            healthCheckFailures: 0,
-            criticalErrors: 0
-        };
-        
-        // Create metrics directory if it doesn't exist
-        if (!fs.existsSync('metrics')) {
-            fs.mkdirSync('metrics');
-        }
-    }
-
-    updateMetrics(type, value) {
-        switch(type) {
-            case 'restart':
-                this.metrics.restarts++;
-                this.metrics.lastRestart = new Date().toISOString();
-                break;
-            case 'memory':
-                this.metrics.memoryUsage.push({
-                    timestamp: new Date().toISOString(),
-                    value: value
-                });
-                // Keep only last 100 measurements
-                if (this.metrics.memoryUsage.length > 100) {
-                    this.metrics.memoryUsage.shift();
-                }
-                break;
-            case 'healthCheck':
-                if (!value) this.metrics.healthCheckFailures++;
-                break;
-            case 'criticalError':
-                this.metrics.criticalErrors++;
-                break;
-        }
-
-        // Calculate uptime percentage
-        const totalTime = Date.now() - this.metrics.startTime;
-        const downtime = this.metrics.restarts * 10000; // Assuming 10s downtime per restart
-        this.metrics.uptimePercentage = ((totalTime - downtime) / totalTime * 100).toFixed(2);
-
-        // Save metrics to file
-        fs.writeFileSync(
-            path.join('metrics', 'metrics.json'),
-            JSON.stringify(this.metrics, null, 2)
-        );
-    }
-}
-
-const metrics = new MetricsTracker();
-
-// Notification system
-const notifier = nodemailer.createTransport(config.notifications.email.smtp);
-
-async function sendNotification(subject, message, critical = false) {
-    if (!config.notifications.email.enabled) return;
-
-    try {
-        await notifier.sendMail({
-            from: config.notifications.email.from,
-            to: config.notifications.email.to,
-            subject: `[${critical ? 'CRITICAL' : 'INFO'}] ${subject}`,
-            text: message
-        });
-        logger.info(`Notification sent: ${subject}`);
-    } catch (error) {
-        logger.error('Failed to send notification:', error);
-    }
-}
-
-// Health check function
-async function performHealthCheck() {
-    try {
-        const response = await fetch(config.monitoring.healthCheckEndpoint);
-        const healthy = response.status === 200;
-        metrics.updateMetrics('healthCheck', healthy);
-        
-        if (!healthy) {
-            logger.warn('Health check failed');
-            sendNotification('Health Check Failed', 'Application health check failed. Investigating...');
-            return false;
-        }
-        return true;
-    } catch (error) {
-        logger.error('Health check error:', error);
-        metrics.updateMetrics('healthCheck', false);
-        return false;
-    }
-}
-
-let yourCommand;
-let isRestarting = false;
-
-async function cleanup() {
-    return new Promise(async (resolve) => {
-        try {
-            if (process.env.CHROME_PID) {
-                exec(`kill ${process.env.CHROME_PID}`);
-            }
-
-            if (yourCommand) {
-                yourCommand.stdout.removeAllListeners('data');
-                yourCommand.kill();
-            }
-
-            await new Promise(resolve => pm2.disconnect(resolve));
-            logger.info('Cleanup completed successfully');
-        } catch (error) {
-            logger.error('Cleanup error:', error);
-        }
-        resolve();
-    });
-}
-
-async function pm2Start() {
-    if (isRestarting) {
-        logger.info('Restart already in progress, skipping...');
-        return;
-    }
-
-    try {
-        isRestarting = true;
-        await cleanup();
-
-        await new Promise((resolve, reject) => {
-            pm2.connect((err) => {
-                if (err) reject(err);
-                else resolve();
-            });
-        });
-
-        // Start the application with config
-        await new Promise((resolve, reject) => {
-            pm2.start(config.app, (err, apps) => {
-                if (err) reject(err);
-                else resolve(apps);
-            });
-        });
-
-        // Start monitoring processes
-        yourCommand = spawn('pm2', ['log']);
-        
-        yourCommand.on('error', (error) => {
-            logger.error('PM2 log process error:', error);
-            metrics.updateMetrics('criticalError');
-            restartWithDelay();
-        });
-
-        yourCommand.on('exit', (code, signal) => {
-            if (code !== 0) {
-                logger.error(`PM2 log process exited with code ${code}, signal: ${signal}`);
-                restartWithDelay();
-            }
-        });
-
-        yourCommand.stdout.on('data', (data) => {
-            const dataStr = data.toString();
-            if (config.monitoring.criticalErrors.some(err => dataStr.includes(err))) {
-                logger.error("Critical error detected in logs:", dataStr);
-                metrics.updateMetrics('criticalError');
-                sendNotification(
-                    'Critical Error Detected',
-                    `Error in application logs: ${dataStr}`,
-                    true
-                );
-                restartWithDelay();
-            }
-        });
-
-        // Start health check interval
-        setInterval(async () => {
-            const isHealthy = await performHealthCheck();
-            if (!isHealthy) restartWithDelay();
-        }, config.monitoring.healthCheckInterval);
-
-        // Start metrics collection interval
-        setInterval(() => {
-            pm2.describe(config.app.name, (err, processDescription) => {
-                if (!err && processDescription[0]) {
-                    metrics.updateMetrics('memory', processDescription[0].monit.memory);
-                }
-            });
-        }, config.monitoring.metricsInterval);
-
-        logger.info('Application started successfully');
-        sendNotification('Application Started', 'The application has been started successfully');
-
-    } catch (error) {
-        logger.error('PM2 start error:', error);
-        metrics.updateMetrics('criticalError');
-        sendNotification(
-            'Application Start Failed',
-            `Failed to start application: ${error.message}`,
-            true
-        );
-        await cleanup();
-        restartWithDelay();
-    } finally {
-        isRestarting = false;
-    }
-}
-
-let restartAttempts = 0;
-
-function restartWithDelay() {
-    if (restartAttempts >= config.monitoring.maxRestartAttempts) {
-        logger.error('Maximum restart attempts reached. Exiting...');
-        sendNotification(
-            'Maximum Restart Attempts Reached',
-            'Application has reached maximum restart attempts and will now exit.',
-            true
-        );
-        process.exit(1);
-    }
-
-    const delay = Math.min(1000 * Math.pow(2, restartAttempts), 30000);
-    restartAttempts++;
-    metrics.updateMetrics('restart');
-    
-    logger.info(`Scheduling restart in ${delay}ms. Attempt ${restartAttempts}/${config.monitoring.maxRestartAttempts}`);
-    setTimeout(pm2Start, delay);
-}
-
-// Process termination handlers
-process.on('SIGTERM', async () => {
-    logger.info('Received SIGTERM. Cleaning up...');
-    await cleanup();
-    process.exit(0);
-});
-
-process.on('SIGINT', async () => {
-    logger.info('Received SIGINT. Cleaning up...');
-    await cleanup();
-    process.exit(0);
-});
-
-// Create necessary directories
-if (!fs.existsSync(config.logging.dir)) {
-    fs.mkdirSync(config.logging.dir);
-}
-
-// Start the application
-pm2Start().catch(error => {
-    logger.error('Fatal error:', error);
-    sendNotification('Fatal Error', `Fatal error occurred: ${error.message}`, true);
-    process.exit(1);
-});

README.md

@@ -0,0 +1,36 @@
+# Prerender PM2 FSCache Server
+
+This runs [prerender](https://github.com/prerender/prerender/) as a PM2 cluster, and efficiently caches the results to the filesystem using [prerender-plugin-fscache](https://github.com/PythonicCafe/prerender-plugin-fscache).
+
+
+```
+bikeshopi-prerender-staging  | 2025-02-21T23:20:11.550Z [fscache] Caching https://b.org/b/x9StaGQ3 to /tmp/prerender-cache/c3/c367c69bb9b5b77b4718f528902984ae23071866
+8f528902984ae23071866
+bikeshopi-prerender-staging  | 2025-02-21T23:20:19.850Z [fscache] Serving https://b.org/b/x9StaGQ3 from /tmp/prerender-cache/c3/c367c69bb9b5b77b4718f528902984ae23071866
+```
+`ecosystem.config.js` and `server.js` are both are mounted from your local filesystem so you can adjust as required     and restart using Docker Compose.
+
+## Environmental Variables for `docker-compose.yml`
+```
+IMAGE=prerender
+CONTAINER_NAME=prerender
+VIRTUAL_HOST=b.org
+LETSENCRYPT_HOST=b.org  # Uses nginxproxy/acme-companion
+PHANTOM_WORKERS=3
+CACHE_VOLUME=prerender-cache  # docker volume create prerender-cache
+
+# optional env variables
+LETSENCRYPT_TEST=false
+PHANTOM_WORKERS=4
+CACHE_STATUS_CODES=200,301,302,303,304,307,308,404
+CACHE_TTL=86400 
+```
+
+## Prender Cache Volume
+The cache is mounted on an external named volume for persistance.
+
+## PM2 Monit
+To monitor PM2 processes, run:
+```
+docker compose exec prerender pm2 monit
+```

docker-compose.yml

@@ -1,40 +1,54 @@
-# prerender server
+# prerender server with pm2 and prerender-plugin-fscache (file system based so 
+# resolves caching issues with pm2 and prerender-memory-cache)
+#
+# docker compose exec prerender pm2 monit
 
 services:
+
   prerender:
     build: .
-    image: ${IMAGE:-bikeshopi/prerender-staging}
+    image: ${IMAGE:-prerender}
-    container_name: ${CONTAINER_NAME:-bikeshopi-prerender-staging} 
+    container_name: ${CONTAINER_NAME:-prerender} 
     user: node
     restart: unless-stopped
-    init: true  # makes all the difference, no more timeout / restarts without chrome alive
+    init: true  # makes all the difference, no more timeout (not always) / restarts without chrome alive
     expose:
       - "3000"
       - "9222"   
     environment:
-      # - PROXY_READ_TIMEOUT=60s
+      - PATH=/home/node/node_modules/pm2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
-      # - PROXY_SET_HEADER_X_REAL_IP=$$remote_addr
-      # - PROXY_SET_HEADER_X_FORWARDED_FOR=$$proxy_add_x_forwarded_for
-      # - PROXY_SET_HEADER_HOST=$$http_host
-      # - PROXY_SET_HEADER_USER_AGENT=$$http_user_agent
       - VIRTUAL_PORT=3000
-      - VIRTUAL_HOST=${VIRTUAL_HOST:-prerender-staging.bikeshopi.org}
+      - VIRTUAL_HOST=${LETSENCRYPT_HOST}
-      - LETSENCRYPT_HOST=${LETSENCRYPT_HOST:-prerender-staging.bikeshopi.org}
+      - LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
-      - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL:-jr@bikeshopi.org}
+      - LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
       - LETSENCRYPT_TEST=${LETSENCRYPT_TEST:-false}
       - CHROME_LOCATION=/usr/bin/chromium
-      - PAGE_LOAD_TIMEOUT=3000  # default 20000 (20 seconds)
+      - PAGE_LOAD_TIMEOUT=${PAGE_LOAD_TIMEOUT:-3000}  # default 20000 (20 seconds)
-      # - CHROME_FLAGS=--headless --disable-gpu --no-sandbox --disable-dev-shm-usage --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222
+      - NODE_ENV=production
-      - COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-staging}     
+      - PHANTOM_WORKERS=${PHANTOM_WORKERS}
+      # prerender-plugin-fscache to the rescue
+      - CACHE_STATUS_CODES=${CACHE_STATUS_CODES:-200,301,302,303,304,307,308,404}
+      - CACHE_TTL=${CACHE_TTL:-86400}
+      - CACHE_PATH=/home/node/prerender-cache
+      - CACHE_VOLUME=${CACHE_VOLUME:-prerender-cache}                          
     networks:
       letsencrypt:
       default:
+    entrypoint: ["pm2-runtime", "start", "ecosystem.config.js"]
     logging:
       driver: "json-file"  
       options:
         max-size: "10m"
-        max-file: "3"      
+        max-file: "3"
+    volumes:
+      - ./ecosystem.config.js:/home/node/ecosystem.config.js
+      - ./server.js:/home/node/server.js
+      - prerender-cache:/home/node/prerender-cache   
 
 networks:
   letsencrypt:
-    external: true
+    external: true
+
+volumes:
+  prerender-cache:
+      name: ${CACHE_VOLUME}

ecosystem.config.js

@@ -0,0 +1,7 @@
+module.exports = [{
+    script: './server.js', // Your main Prerender server
+    name: 'prerender-app',
+    cwd: '/home/node',
+    exec_mode: 'cluster',
+    instances: process.env.PHANTOM_WORKERS || 3, // Adjust based on your CPU cores
+}];

server.js

@@ -14,7 +14,8 @@ const server = prerender({
     logRequests: true,
     browserDebuggingPort: 9222,
 });
-server.use(require('prerender-memory-cache'))
+
+server.use(require("prerender-plugin-fscache")); // allows pm2 instances to live happily since written to filesystem
 server.use(prerender.httpHeaders());
 server.use(prerender.removeScriptTags());
 server.use(prerender.sendPrerenderHeader());