bike/Prerender-PM2-FSCache-Server
1
0
Fork 0
Code Issues Pull Requests Projects Releases Wiki Activity

Complete overhaul - changed to pm2-runtime and the excellent prerender-plugin-fscache. Working excellently!

Browse Source
This commit is contained in:
Jonathan Rosenbaum 2025-02-22 04:05:47 -05:00
parent 85336a52f9
commit b28d11474e
7 changed files with 82 additions and 363 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
.dockerignore Normal file
View File

@ -0,0 +1 @@
.env

10
Dockerfile
View File

@ -17,15 +17,15 @@ ENV CHROME_FLAGS="--headless --disable-gpu --no-sandbox --disable-dev-shm-usage
WORKDIR /home/node
USER root
RUN mkdir -p /home/node/prerender-cache && chown -R node:node /home/node/prerender-cache
RUN apt-get update && apt-get install -y \
chromium \
--no-install-recommends \
&& rm -rf /var/lib/apt/lists/*
# Already has git
# RUN apt-get update -y
RUN yarn add prerender prerender-memory-cache pm2 winston nodemailer
USER node:node
RUN yarn add prerender pm2 prerender-plugin-fscache
ENTRYPOINT ["node", "server.js"]
ENTRYPOINT ["pm2-runtime", "start", "ecosystem.config.js"]

340
PM2Manager
View File

@ -1,340 +0,0 @@
// To use this enhanced version, you'll need to:
// Install additional dependencies:
// npm install winston nodemailer node-fetch
// Set up environment variables:
// export NOTIFICATION_EMAIL=alerts@yourdomain.com
// export ADMIN_EMAIL=admin@yourdomain.com
// export SMTP_HOST=smtp.yourdomain.com
// export SMTP_PORT=587
// export SMTP_USER=your-user
// export SMTP_PASS=your-password
const { spawn, exec } = require('child_process');
const pm2 = require('pm2');
const fs = require('fs');
const path = require('path');
const winston = require('winston');
const nodemailer = require('nodemailer');
// Configuration file
const config = {
app: {
name: 'server',
script: 'server.js',
instances: 4,
max_memory_restart: '300M',
max_restarts: 10,
exp_backoff_restart_delay: 100
},
monitoring: {
healthCheckInterval: 30000,
healthCheckEndpoint: 'http://localhost:3000/health',
metricsInterval: 60000,
maxRestartAttempts: 5,
criticalErrors: ['page timed out', 'parse html timed out', 'Timed out waiting for']
},
notifications: {
email: {
enabled: true,
from: process.env.NOTIFICATION_EMAIL,
to: process.env.ADMIN_EMAIL,
smtp: {
host: process.env.SMTP_HOST,
port: process.env.SMTP_PORT,
auth: {
user: process.env.SMTP_USER,
pass: process.env.SMTP_PASS
}
}
}
},
logging: {
dir: 'logs',
maxSize: '10m',
maxFiles: '7d'
}
};
// Save config to file
fs.writeFileSync('pm2-config.json', JSON.stringify(config, null, 2));
// Initialize logger
const logger = winston.createLogger({
level: 'info',
format: winston.format.combine(
winston.format.timestamp(),
winston.format.json()
),
transports: [
new winston.transports.Console(),
new winston.transports.File({
filename: path.join(config.logging.dir, 'error.log'),
level: 'error',
maxsize: 5242880, // 5MB
maxFiles: 5
}),
new winston.transports.File({
filename: path.join(config.logging.dir, 'combined.log'),
maxsize: 5242880,
maxFiles: 5
})
]
});
// Metrics tracking
class MetricsTracker {
constructor() {
this.metrics = {
startTime: Date.now(),
restarts: 0,
lastRestart: null,
uptimePercentage: 100,
memoryUsage: [],
healthCheckFailures: 0,
criticalErrors: 0
};
// Create metrics directory if it doesn't exist
if (!fs.existsSync('metrics')) {
fs.mkdirSync('metrics');
}
}
updateMetrics(type, value) {
switch(type) {
case 'restart':
this.metrics.restarts++;
this.metrics.lastRestart = new Date().toISOString();
break;
case 'memory':
this.metrics.memoryUsage.push({
timestamp: new Date().toISOString(),
value: value
});
// Keep only last 100 measurements
if (this.metrics.memoryUsage.length > 100) {
this.metrics.memoryUsage.shift();
}
break;
case 'healthCheck':
if (!value) this.metrics.healthCheckFailures++;
break;
case 'criticalError':
this.metrics.criticalErrors++;
break;
}
// Calculate uptime percentage
const totalTime = Date.now() - this.metrics.startTime;
const downtime = this.metrics.restarts * 10000; // Assuming 10s downtime per restart
this.metrics.uptimePercentage = ((totalTime - downtime) / totalTime * 100).toFixed(2);
// Save metrics to file
fs.writeFileSync(
path.join('metrics', 'metrics.json'),
JSON.stringify(this.metrics, null, 2)
);
}
}
const metrics = new MetricsTracker();
// Notification system
const notifier = nodemailer.createTransport(config.notifications.email.smtp);
async function sendNotification(subject, message, critical = false) {
if (!config.notifications.email.enabled) return;
try {
await notifier.sendMail({
from: config.notifications.email.from,
to: config.notifications.email.to,
subject: `[${critical ? 'CRITICAL' : 'INFO'}] ${subject}`,
text: message
});
logger.info(`Notification sent: ${subject}`);
} catch (error) {
logger.error('Failed to send notification:', error);
}
}
// Health check function
async function performHealthCheck() {
try {
const response = await fetch(config.monitoring.healthCheckEndpoint);
const healthy = response.status === 200;
metrics.updateMetrics('healthCheck', healthy);
if (!healthy) {
logger.warn('Health check failed');
sendNotification('Health Check Failed', 'Application health check failed. Investigating...');
return false;
}
return true;
} catch (error) {
logger.error('Health check error:', error);
metrics.updateMetrics('healthCheck', false);
return false;
}
}
let yourCommand;
let isRestarting = false;
async function cleanup() {
return new Promise(async (resolve) => {
try {
if (process.env.CHROME_PID) {
exec(`kill ${process.env.CHROME_PID}`);
}
if (yourCommand) {
yourCommand.stdout.removeAllListeners('data');
yourCommand.kill();
}
await new Promise(resolve => pm2.disconnect(resolve));
logger.info('Cleanup completed successfully');
} catch (error) {
logger.error('Cleanup error:', error);
}
resolve();
});
}
async function pm2Start() {
if (isRestarting) {
logger.info('Restart already in progress, skipping...');
return;
}
try {
isRestarting = true;
await cleanup();
await new Promise((resolve, reject) => {
pm2.connect((err) => {
if (err) reject(err);
else resolve();
});
});
// Start the application with config
await new Promise((resolve, reject) => {
pm2.start(config.app, (err, apps) => {
if (err) reject(err);
else resolve(apps);
});
});
// Start monitoring processes
yourCommand = spawn('pm2', ['log']);
yourCommand.on('error', (error) => {
logger.error('PM2 log process error:', error);
metrics.updateMetrics('criticalError');
restartWithDelay();
});
yourCommand.on('exit', (code, signal) => {
if (code !== 0) {
logger.error(`PM2 log process exited with code ${code}, signal: ${signal}`);
restartWithDelay();
}
});
yourCommand.stdout.on('data', (data) => {
const dataStr = data.toString();
if (config.monitoring.criticalErrors.some(err => dataStr.includes(err))) {
logger.error("Critical error detected in logs:", dataStr);
metrics.updateMetrics('criticalError');
sendNotification(
'Critical Error Detected',
`Error in application logs: ${dataStr}`,
true
);
restartWithDelay();
}
});
// Start health check interval
setInterval(async () => {
const isHealthy = await performHealthCheck();
if (!isHealthy) restartWithDelay();
}, config.monitoring.healthCheckInterval);
// Start metrics collection interval
setInterval(() => {
pm2.describe(config.app.name, (err, processDescription) => {
if (!err && processDescription[0]) {
metrics.updateMetrics('memory', processDescription[0].monit.memory);
}
});
}, config.monitoring.metricsInterval);
logger.info('Application started successfully');
sendNotification('Application Started', 'The application has been started successfully');
} catch (error) {
logger.error('PM2 start error:', error);
metrics.updateMetrics('criticalError');
sendNotification(
'Application Start Failed',
`Failed to start application: ${error.message}`,
true
);
await cleanup();
restartWithDelay();
} finally {
isRestarting = false;
}
}
let restartAttempts = 0;
function restartWithDelay() {
if (restartAttempts >= config.monitoring.maxRestartAttempts) {
logger.error('Maximum restart attempts reached. Exiting...');
sendNotification(
'Maximum Restart Attempts Reached',
'Application has reached maximum restart attempts and will now exit.',
true
);
process.exit(1);
}
const delay = Math.min(1000 * Math.pow(2, restartAttempts), 30000);
restartAttempts++;
metrics.updateMetrics('restart');
logger.info(`Scheduling restart in ${delay}ms. Attempt ${restartAttempts}/${config.monitoring.maxRestartAttempts}`);
setTimeout(pm2Start, delay);
}
// Process termination handlers
process.on('SIGTERM', async () => {
logger.info('Received SIGTERM. Cleaning up...');
await cleanup();
process.exit(0);
});
process.on('SIGINT', async () => {
logger.info('Received SIGINT. Cleaning up...');
await cleanup();
process.exit(0);
});
// Create necessary directories
if (!fs.existsSync(config.logging.dir)) {
fs.mkdirSync(config.logging.dir);
}
// Start the application
pm2Start().catch(error => {
logger.error('Fatal error:', error);
sendNotification('Fatal Error', `Fatal error occurred: ${error.message}`, true);
process.exit(1);
});

36
README.md
View File

@ -0,0 +1,36 @@
# Prerender PM2 FSCache Server
This runs [prerender](https://github.com/prerender/prerender/) as a PM2 cluster, and efficiently caches the results to the filesystem using [prerender-plugin-fscache](https://github.com/PythonicCafe/prerender-plugin-fscache).
```
bikeshopi-prerender-staging | 2025-02-21T23:20:11.550Z [fscache] Caching https://b.org/b/x9StaGQ3 to /tmp/prerender-cache/c3/c367c69bb9b5b77b4718f528902984ae23071866
8f528902984ae23071866
bikeshopi-prerender-staging | 2025-02-21T23:20:19.850Z [fscache] Serving https://b.org/b/x9StaGQ3 from /tmp/prerender-cache/c3/c367c69bb9b5b77b4718f528902984ae23071866
```
`ecosystem.config.js` and `server.js` are both are mounted from your local filesystem so you can adjust as required and restart using Docker Compose.
## Environmental Variables for `docker-compose.yml`
```
IMAGE=prerender
CONTAINER_NAME=prerender
VIRTUAL_HOST=b.org
LETSENCRYPT_HOST=b.org # Uses nginxproxy/acme-companion
PHANTOM_WORKERS=3
CACHE_VOLUME=prerender-cache # docker volume create prerender-cache
# optional env variables
LETSENCRYPT_TEST=false
PHANTOM_WORKERS=4
CACHE_STATUS_CODES=200,301,302,303,304,307,308,404
CACHE_TTL=86400
```
## Prender Cache Volume
The cache is mounted on an external named volume for persistance.
## PM2 Monit
To monitor PM2 processes, run:
```
docker compose exec prerender pm2 monit
```

44
docker-compose.yml
View File

@ -1,40 +1,54 @@
# prerender server
# prerender server with pm2 and prerender-plugin-fscache (file system based so
# resolves caching issues with pm2 and prerender-memory-cache)
#
# docker compose exec prerender pm2 monit
services:
prerender:
build: .
image: ${IMAGE:-bikeshopi/prerender-staging}
container_name: ${CONTAINER_NAME:-bikeshopi-prerender-staging}
image: ${IMAGE:-prerender}
container_name: ${CONTAINER_NAME:-prerender}
user: node
restart: unless-stopped
init: true # makes all the difference, no more timeout / restarts without chrome alive
init: true # makes all the difference, no more timeout (not always) / restarts without chrome alive
expose:
- "3000"
- "9222"
environment:
# - PROXY_READ_TIMEOUT=60s
# - PROXY_SET_HEADER_X_REAL_IP=$$remote_addr
# - PROXY_SET_HEADER_X_FORWARDED_FOR=$$proxy_add_x_forwarded_for
# - PROXY_SET_HEADER_HOST=$$http_host
# - PROXY_SET_HEADER_USER_AGENT=$$http_user_agent
- PATH=/home/node/node_modules/pm2/bin:/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin
- VIRTUAL_PORT=3000
- VIRTUAL_HOST=${VIRTUAL_HOST:-prerender-staging.bikeshopi.org}
- LETSENCRYPT_HOST=${LETSENCRYPT_HOST:-prerender-staging.bikeshopi.org}
- LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL:-jr@bikeshopi.org}
- VIRTUAL_HOST=${LETSENCRYPT_HOST}
- LETSENCRYPT_HOST=${LETSENCRYPT_HOST}
- LETSENCRYPT_EMAIL=${LETSENCRYPT_EMAIL}
- LETSENCRYPT_TEST=${LETSENCRYPT_TEST:-false}
- CHROME_LOCATION=/usr/bin/chromium
- PAGE_LOAD_TIMEOUT=3000 # default 20000 (20 seconds)
# - CHROME_FLAGS=--headless --disable-gpu --no-sandbox --disable-dev-shm-usage --remote-debugging-address=0.0.0.0 --remote-debugging-port=9222
- COMPOSE_PROJECT_NAME=${COMPOSE_PROJECT_NAME:-staging}
- PAGE_LOAD_TIMEOUT=${PAGE_LOAD_TIMEOUT:-3000} # default 20000 (20 seconds)
- NODE_ENV=production
- PHANTOM_WORKERS=${PHANTOM_WORKERS}
# prerender-plugin-fscache to the rescue
- CACHE_STATUS_CODES=${CACHE_STATUS_CODES:-200,301,302,303,304,307,308,404}
- CACHE_TTL=${CACHE_TTL:-86400}
- CACHE_PATH=/home/node/prerender-cache
- CACHE_VOLUME=${CACHE_VOLUME:-prerender-cache}
networks:
letsencrypt:
default:
entrypoint: ["pm2-runtime", "start", "ecosystem.config.js"]
logging:
driver: "json-file"
options:
max-size: "10m"
max-file: "3"
volumes:
- ./ecosystem.config.js:/home/node/ecosystem.config.js
- ./server.js:/home/node/server.js
- prerender-cache:/home/node/prerender-cache
networks:
letsencrypt:
external: true
volumes:
prerender-cache:
name: ${CACHE_VOLUME}

7
ecosystem.config.js Normal file
View File

@ -0,0 +1,7 @@
module.exports = [{
script: './server.js', // Your main Prerender server
name: 'prerender-app',
cwd: '/home/node',
exec_mode: 'cluster',
instances: process.env.PHANTOM_WORKERS || 3, // Adjust based on your CPU cores
}];

3
server.js
View File

@ -14,7 +14,8 @@ const server = prerender({
logRequests: true,
browserDebuggingPort: 9222,
});
server.use(require('prerender-memory-cache'))
server.use(require("prerender-plugin-fscache")); // allows pm2 instances to live happily since written to filesystem
server.use(prerender.httpHeaders());
server.use(prerender.removeScriptTags());
server.use(prerender.sendPrerenderHeader());