First Commit!
This commit is contained in:
+340
@@ -0,0 +1,340 @@
|
||||
// To use this enhanced version, you'll need to:
|
||||
|
||||
// Install additional dependencies:
|
||||
// npm install winston nodemailer node-fetch
|
||||
|
||||
// Set up environment variables:
|
||||
// export NOTIFICATION_EMAIL=alerts@yourdomain.com
|
||||
// export ADMIN_EMAIL=admin@yourdomain.com
|
||||
// export SMTP_HOST=smtp.yourdomain.com
|
||||
// export SMTP_PORT=587
|
||||
// export SMTP_USER=your-user
|
||||
// export SMTP_PASS=your-password
|
||||
|
||||
|
||||
const { spawn, exec } = require('child_process');
|
||||
const pm2 = require('pm2');
|
||||
const fs = require('fs');
|
||||
const path = require('path');
|
||||
const winston = require('winston');
|
||||
const nodemailer = require('nodemailer');
|
||||
|
||||
// Configuration file
|
||||
const config = {
|
||||
app: {
|
||||
name: 'server',
|
||||
script: 'server.js',
|
||||
instances: 4,
|
||||
max_memory_restart: '300M',
|
||||
max_restarts: 10,
|
||||
exp_backoff_restart_delay: 100
|
||||
},
|
||||
monitoring: {
|
||||
healthCheckInterval: 30000,
|
||||
healthCheckEndpoint: 'http://localhost:3000/health',
|
||||
metricsInterval: 60000,
|
||||
maxRestartAttempts: 5,
|
||||
criticalErrors: ['page timed out', 'parse html timed out', 'Timed out waiting for']
|
||||
},
|
||||
notifications: {
|
||||
email: {
|
||||
enabled: true,
|
||||
from: process.env.NOTIFICATION_EMAIL,
|
||||
to: process.env.ADMIN_EMAIL,
|
||||
smtp: {
|
||||
host: process.env.SMTP_HOST,
|
||||
port: process.env.SMTP_PORT,
|
||||
auth: {
|
||||
user: process.env.SMTP_USER,
|
||||
pass: process.env.SMTP_PASS
|
||||
}
|
||||
}
|
||||
}
|
||||
},
|
||||
logging: {
|
||||
dir: 'logs',
|
||||
maxSize: '10m',
|
||||
maxFiles: '7d'
|
||||
}
|
||||
};
|
||||
|
||||
// Save config to file
|
||||
fs.writeFileSync('pm2-config.json', JSON.stringify(config, null, 2));
|
||||
|
||||
// Initialize logger
|
||||
const logger = winston.createLogger({
|
||||
level: 'info',
|
||||
format: winston.format.combine(
|
||||
winston.format.timestamp(),
|
||||
winston.format.json()
|
||||
),
|
||||
transports: [
|
||||
new winston.transports.Console(),
|
||||
new winston.transports.File({
|
||||
filename: path.join(config.logging.dir, 'error.log'),
|
||||
level: 'error',
|
||||
maxsize: 5242880, // 5MB
|
||||
maxFiles: 5
|
||||
}),
|
||||
new winston.transports.File({
|
||||
filename: path.join(config.logging.dir, 'combined.log'),
|
||||
maxsize: 5242880,
|
||||
maxFiles: 5
|
||||
})
|
||||
]
|
||||
});
|
||||
|
||||
// Metrics tracking
|
||||
class MetricsTracker {
|
||||
constructor() {
|
||||
this.metrics = {
|
||||
startTime: Date.now(),
|
||||
restarts: 0,
|
||||
lastRestart: null,
|
||||
uptimePercentage: 100,
|
||||
memoryUsage: [],
|
||||
healthCheckFailures: 0,
|
||||
criticalErrors: 0
|
||||
};
|
||||
|
||||
// Create metrics directory if it doesn't exist
|
||||
if (!fs.existsSync('metrics')) {
|
||||
fs.mkdirSync('metrics');
|
||||
}
|
||||
}
|
||||
|
||||
updateMetrics(type, value) {
|
||||
switch(type) {
|
||||
case 'restart':
|
||||
this.metrics.restarts++;
|
||||
this.metrics.lastRestart = new Date().toISOString();
|
||||
break;
|
||||
case 'memory':
|
||||
this.metrics.memoryUsage.push({
|
||||
timestamp: new Date().toISOString(),
|
||||
value: value
|
||||
});
|
||||
// Keep only last 100 measurements
|
||||
if (this.metrics.memoryUsage.length > 100) {
|
||||
this.metrics.memoryUsage.shift();
|
||||
}
|
||||
break;
|
||||
case 'healthCheck':
|
||||
if (!value) this.metrics.healthCheckFailures++;
|
||||
break;
|
||||
case 'criticalError':
|
||||
this.metrics.criticalErrors++;
|
||||
break;
|
||||
}
|
||||
|
||||
// Calculate uptime percentage
|
||||
const totalTime = Date.now() - this.metrics.startTime;
|
||||
const downtime = this.metrics.restarts * 10000; // Assuming 10s downtime per restart
|
||||
this.metrics.uptimePercentage = ((totalTime - downtime) / totalTime * 100).toFixed(2);
|
||||
|
||||
// Save metrics to file
|
||||
fs.writeFileSync(
|
||||
path.join('metrics', 'metrics.json'),
|
||||
JSON.stringify(this.metrics, null, 2)
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const metrics = new MetricsTracker();
|
||||
|
||||
// Notification system
|
||||
const notifier = nodemailer.createTransport(config.notifications.email.smtp);
|
||||
|
||||
async function sendNotification(subject, message, critical = false) {
|
||||
if (!config.notifications.email.enabled) return;
|
||||
|
||||
try {
|
||||
await notifier.sendMail({
|
||||
from: config.notifications.email.from,
|
||||
to: config.notifications.email.to,
|
||||
subject: `[${critical ? 'CRITICAL' : 'INFO'}] ${subject}`,
|
||||
text: message
|
||||
});
|
||||
logger.info(`Notification sent: ${subject}`);
|
||||
} catch (error) {
|
||||
logger.error('Failed to send notification:', error);
|
||||
}
|
||||
}
|
||||
|
||||
// Health check function
|
||||
async function performHealthCheck() {
|
||||
try {
|
||||
const response = await fetch(config.monitoring.healthCheckEndpoint);
|
||||
const healthy = response.status === 200;
|
||||
metrics.updateMetrics('healthCheck', healthy);
|
||||
|
||||
if (!healthy) {
|
||||
logger.warn('Health check failed');
|
||||
sendNotification('Health Check Failed', 'Application health check failed. Investigating...');
|
||||
return false;
|
||||
}
|
||||
return true;
|
||||
} catch (error) {
|
||||
logger.error('Health check error:', error);
|
||||
metrics.updateMetrics('healthCheck', false);
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
let yourCommand;
|
||||
let isRestarting = false;
|
||||
|
||||
async function cleanup() {
|
||||
return new Promise(async (resolve) => {
|
||||
try {
|
||||
if (process.env.CHROME_PID) {
|
||||
exec(`kill ${process.env.CHROME_PID}`);
|
||||
}
|
||||
|
||||
if (yourCommand) {
|
||||
yourCommand.stdout.removeAllListeners('data');
|
||||
yourCommand.kill();
|
||||
}
|
||||
|
||||
await new Promise(resolve => pm2.disconnect(resolve));
|
||||
logger.info('Cleanup completed successfully');
|
||||
} catch (error) {
|
||||
logger.error('Cleanup error:', error);
|
||||
}
|
||||
resolve();
|
||||
});
|
||||
}
|
||||
|
||||
async function pm2Start() {
|
||||
if (isRestarting) {
|
||||
logger.info('Restart already in progress, skipping...');
|
||||
return;
|
||||
}
|
||||
|
||||
try {
|
||||
isRestarting = true;
|
||||
await cleanup();
|
||||
|
||||
await new Promise((resolve, reject) => {
|
||||
pm2.connect((err) => {
|
||||
if (err) reject(err);
|
||||
else resolve();
|
||||
});
|
||||
});
|
||||
|
||||
// Start the application with config
|
||||
await new Promise((resolve, reject) => {
|
||||
pm2.start(config.app, (err, apps) => {
|
||||
if (err) reject(err);
|
||||
else resolve(apps);
|
||||
});
|
||||
});
|
||||
|
||||
// Start monitoring processes
|
||||
yourCommand = spawn('pm2', ['log']);
|
||||
|
||||
yourCommand.on('error', (error) => {
|
||||
logger.error('PM2 log process error:', error);
|
||||
metrics.updateMetrics('criticalError');
|
||||
restartWithDelay();
|
||||
});
|
||||
|
||||
yourCommand.on('exit', (code, signal) => {
|
||||
if (code !== 0) {
|
||||
logger.error(`PM2 log process exited with code ${code}, signal: ${signal}`);
|
||||
restartWithDelay();
|
||||
}
|
||||
});
|
||||
|
||||
yourCommand.stdout.on('data', (data) => {
|
||||
const dataStr = data.toString();
|
||||
if (config.monitoring.criticalErrors.some(err => dataStr.includes(err))) {
|
||||
logger.error("Critical error detected in logs:", dataStr);
|
||||
metrics.updateMetrics('criticalError');
|
||||
sendNotification(
|
||||
'Critical Error Detected',
|
||||
`Error in application logs: ${dataStr}`,
|
||||
true
|
||||
);
|
||||
restartWithDelay();
|
||||
}
|
||||
});
|
||||
|
||||
// Start health check interval
|
||||
setInterval(async () => {
|
||||
const isHealthy = await performHealthCheck();
|
||||
if (!isHealthy) restartWithDelay();
|
||||
}, config.monitoring.healthCheckInterval);
|
||||
|
||||
// Start metrics collection interval
|
||||
setInterval(() => {
|
||||
pm2.describe(config.app.name, (err, processDescription) => {
|
||||
if (!err && processDescription[0]) {
|
||||
metrics.updateMetrics('memory', processDescription[0].monit.memory);
|
||||
}
|
||||
});
|
||||
}, config.monitoring.metricsInterval);
|
||||
|
||||
logger.info('Application started successfully');
|
||||
sendNotification('Application Started', 'The application has been started successfully');
|
||||
|
||||
} catch (error) {
|
||||
logger.error('PM2 start error:', error);
|
||||
metrics.updateMetrics('criticalError');
|
||||
sendNotification(
|
||||
'Application Start Failed',
|
||||
`Failed to start application: ${error.message}`,
|
||||
true
|
||||
);
|
||||
await cleanup();
|
||||
restartWithDelay();
|
||||
} finally {
|
||||
isRestarting = false;
|
||||
}
|
||||
}
|
||||
|
||||
let restartAttempts = 0;
|
||||
|
||||
function restartWithDelay() {
|
||||
if (restartAttempts >= config.monitoring.maxRestartAttempts) {
|
||||
logger.error('Maximum restart attempts reached. Exiting...');
|
||||
sendNotification(
|
||||
'Maximum Restart Attempts Reached',
|
||||
'Application has reached maximum restart attempts and will now exit.',
|
||||
true
|
||||
);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const delay = Math.min(1000 * Math.pow(2, restartAttempts), 30000);
|
||||
restartAttempts++;
|
||||
metrics.updateMetrics('restart');
|
||||
|
||||
logger.info(`Scheduling restart in ${delay}ms. Attempt ${restartAttempts}/${config.monitoring.maxRestartAttempts}`);
|
||||
setTimeout(pm2Start, delay);
|
||||
}
|
||||
|
||||
// Process termination handlers
|
||||
process.on('SIGTERM', async () => {
|
||||
logger.info('Received SIGTERM. Cleaning up...');
|
||||
await cleanup();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
process.on('SIGINT', async () => {
|
||||
logger.info('Received SIGINT. Cleaning up...');
|
||||
await cleanup();
|
||||
process.exit(0);
|
||||
});
|
||||
|
||||
// Create necessary directories
|
||||
if (!fs.existsSync(config.logging.dir)) {
|
||||
fs.mkdirSync(config.logging.dir);
|
||||
}
|
||||
|
||||
// Start the application
|
||||
pm2Start().catch(error => {
|
||||
logger.error('Fatal error:', error);
|
||||
sendNotification('Fatal Error', `Fatal error occurred: ${error.message}`, true);
|
||||
process.exit(1);
|
||||
});
|
||||
Reference in New Issue
Block a user