Performance Testing Expert

A specialized agent for implementing comprehensive performance testing strategies including load testing, stress testing, endurance testing, and performance monitoring with modern tools and methodologies.

Core Capabilities

Performance Testing Types

• Load Testing: Normal expected load conditions
• Stress Testing: Beyond normal capacity limits
• Spike Testing: Sudden load increases
• Endurance Testing: Extended periods under load
• Volume Testing: Large amounts of data

Performance Metrics

• Response time and latency
• Throughput and requests per second
• Resource utilization (CPU, memory, disk, network)
• Error rates and availability
• Scalability and bottleneck identification

Tools & Technologies

• Load Testing: K6, JMeter, Artillery, Gatling
• Monitoring: Prometheus, Grafana, New Relic, DataDog
• Profiling: Chrome DevTools, Node.js profiler, py-spy
• APM: Application Performance Monitoring solutions

Load Testing Implementations

K6 Load Testing Suite

// k6-tests/load-test-suite.js
import http from 'k6/http';
import { check, sleep } from 'k6';
import { Counter, Rate, Trend, Gauge } from 'k6/metrics';

// Custom metrics
const httpReqFailed = new Rate('http_req_failed');
const httpReqDuration = new Trend('http_req_duration');
const activeUsers = new Gauge('active_users');
const dataReceived = new Counter('data_received');

// Test configuration
export const options = {
  stages: [
    // Ramp-up
    { duration: '2m', target: 10 },   // Warm up
    { duration: '5m', target: 50 },   // Normal load
    { duration: '10m', target: 100 }, // Peak load
    { duration: '5m', target: 200 },  // Stress test
    { duration: '2m', target: 0 },    // Ramp-down
  ],
  
  thresholds: {
    // Response time thresholds
    http_req_duration: [
      'p(50)<500',   // 50% of requests under 500ms
      'p(90)<1000',  // 90% of requests under 1s
      'p(95)<2000',  // 95% of requests under 2s
    ],
    
    // Error rate thresholds
    http_req_failed: ['rate<0.01'], // Error rate under 1%
    
    // Throughput thresholds
    http_reqs: ['rate>100'], // At least 100 RPS
  },
};

// Test data
const BASE_URL = __ENV.BASE_URL || 'http://localhost:3000';
const API_TOKEN = __ENV.API_TOKEN || 'test-token';

const users = [
  { email: 'user1@test.com', password: 'password123' },
  { email: 'user2@test.com', password: 'password123' },
  { email: 'user3@test.com', password: 'password123' },
];

export function setup() {
  // Setup test data
  console.log('Setting up test data...');
  
  // Create test users
  users.forEach(user => {
    const response = http.post(`${BASE_URL}/api/auth/register`, 
      JSON.stringify(user), 
      { headers: { 'Content-Type': 'application/json' } }
    );
    
    if (response.status !== 201) {
      console.warn(`Failed to create user ${user.email}`);
    }
  });
  
  return { users: users };
}

export default function(data) {
  const user = data.users[Math.floor(Math.random() * data.users.length)];
  
  // Authentication flow
  const authResponse = authenticate(user);
  if (!authResponse.token) {
    console.error('Authentication failed');
    return;
  }
  
  // User journey scenarios
  const scenario = Math.random();
  
  if (scenario < 0.4) {
    // 40% - Browse products
    browseProducts(authResponse.token);
  } else if (scenario < 0.7) {
    // 30% - Create and manage orders
    orderWorkflow(authResponse.token);
  } else if (scenario < 0.9) {
    // 20% - User profile operations
    userProfileOperations(authResponse.token);
  } else {
    // 10% - Admin operations (if applicable)
    adminOperations(authResponse.token);
  }
  
  // Think time between requests
  sleep(Math.random() * 3 + 1); // 1-4 seconds
}

function authenticate(user) {
  const response = http.post(`${BASE_URL}/api/auth/login`, 
    JSON.stringify({ 
      email: user.email, 
      password: user.password 
    }), 
    { 
      headers: { 'Content-Type': 'application/json' },
      tags: { name: 'auth_login' }
    }
  );
  
  check(response, {
    'authentication successful': (r) => r.status === 200,
    'token received': (r) => r.json('token') !== null,
  });
  
  httpReqFailed.add(response.status >= 400);
  httpReqDuration.add(response.timings.duration);
  dataReceived.add(response.body.length);
  
  return response.json();
}

function browseProducts(token) {
  const headers = { 
    'Authorization': `Bearer ${token}`,
    'Content-Type': 'application/json'
  };
  
  // Get products list
  const productsResponse = http.get(`${BASE_URL}/api/products`, 
    { headers, tags: { name: 'get_products' } }
  );
  
  check(productsResponse, {
    'products loaded': (r) => r.status === 200,
    'products count > 0': (r) => r.json('data').length > 0,
  });
  
  if (productsResponse.status === 200) {
    const products = productsResponse.json('data');
    
    // View random product details
    if (products.length > 0) {
      const randomProduct = products[Math.floor(Math.random() * products.length)];
      
      const productResponse = http.get(`${BASE_URL}/api/products/${randomProduct.id}`, 
        { headers, tags: { name: 'get_product_detail' } }
      );
      
      check(productResponse, {
        'product detail loaded': (r) => r.status === 200,
      });
    }
  }
  
  sleep(1);
}

function orderWorkflow(token) {
  const headers = { 
    'Authorization': `Bearer ${token}`,
    'Content-Type': 'application/json'
  };
  
  // Create order
  const createOrderResponse = http.post(`${BASE_URL}/api/orders`, 
    JSON.stringify({}), 
    { headers, tags: { name: 'create_order' } }
  );
  
  check(createOrderResponse, {
    'order created': (r) => r.status === 201,
  });
  
  if (createOrderResponse.status === 201) {
    const order = createOrderResponse.json();
    
    // Add items to order
    const addItemResponse = http.post(`${BASE_URL}/api/orders/${order.id}/items`, 
      JSON.stringify({
        productId: '1',
        quantity: Math.floor(Math.random() * 5) + 1
      }), 
      { headers, tags: { name: 'add_order_item' } }
    );
    
    check(addItemResponse, {
      'item added to order': (r) => r.status === 200,
    });
    
    // Get order details
    const orderDetailsResponse = http.get(`${BASE_URL}/api/orders/${order.id}`, 
      { headers, tags: { name: 'get_order_details' } }
    );
    
    check(orderDetailsResponse, {
      'order details retrieved': (r) => r.status === 200,
    });
    
    // Simulate order confirmation (30% chance)
    if (Math.random() < 0.3) {
      const confirmResponse = http.post(`${BASE_URL}/api/orders/${order.id}/confirm`, 
        JSON.stringify({ paymentMethod: 'credit_card' }), 
        { headers, tags: { name: 'confirm_order' } }
      );
      
      check(confirmResponse, {
        'order confirmed': (r) => r.status === 200,
      });
    }
  }
  
  sleep(2);
}

function userProfileOperations(token) {
  const headers = { 
    'Authorization': `Bearer ${token}`,
    'Content-Type': 'application/json'
  };
  
  // Get user profile
  const profileResponse = http.get(`${BASE_URL}/api/users/profile`, 
    { headers, tags: { name: 'get_user_profile' } }
  );
  
  check(profileResponse, {
    'profile loaded': (r) => r.status === 200,
  });
  
  // Update profile (20% chance)
  if (Math.random() < 0.2) {
    const updateResponse = http.put(`${BASE_URL}/api/users/profile`, 
      JSON.stringify({
        name: `Updated User ${Date.now()}`,
        preferences: { theme: 'dark' }
      }), 
      { headers, tags: { name: 'update_user_profile' } }
    );
    
    check(updateResponse, {
      'profile updated': (r) => r.status === 200,
    });
  }
  
  sleep(1);
}

function adminOperations(token) {
  const headers = { 
    'Authorization': `Bearer ${token}`,
    'Content-Type': 'application/json'
  };
  
  // Get analytics data
  const analyticsResponse = http.get(`${BASE_URL}/api/admin/analytics`, 
    { headers, tags: { name: 'get_analytics' } }
  );
  
  check(analyticsResponse, {
    'analytics loaded': (r) => r.status === 200 || r.status === 403,
  });
  
  sleep(1);
}

export function teardown(data) {
  console.log('Cleaning up test data...');
  // Cleanup operations if needed
}

JMeter Test Plan Configuration

<!-- jmeter-test-plan.jmx -->
<?xml version="1.0" encoding="UTF-8"?>
<jmeterTestPlan version="1.2" properties="5.0" jmeter="5.5">
  <hashTree>
    <TestPlan guiclass="TestPlanGui" testclass="TestPlan" testname="E-commerce Performance Test">
      <stringProp name="TestPlan.comments">Comprehensive performance test for e-commerce application</stringProp>
      <boolProp name="TestPlan.functional_mode">false</boolProp>
      <boolProp name="TestPlan.tearDown_on_shutdown">true</boolProp>
      <boolProp name="TestPlan.serialize_threadgroups">false</boolProp>
      <elementProp name="TestPlan.arguments" elementType="Arguments">
        <collectionProp name="Arguments.arguments">
          <elementProp name="base_url" elementType="Argument">
            <stringProp name="Argument.name">base_url</stringProp>
            <stringProp name="Argument.value">${__P(base_url,http://localhost:3000)}</stringProp>
          </elementProp>
        </collectionProp>
      </elementProp>
    </TestPlan>
    <hashTree>
      <!-- Thread Group for Load Test -->
      <ThreadGroup guiclass="ThreadGroupGui" testclass="ThreadGroup" testname="Load Test Users">
        <stringProp name="ThreadGroup.on_sample_error">continue</stringProp>
        <elementProp name="ThreadGroup.main_controller" elementType="LoopController">
          <boolProp name="LoopController.continue_forever">false</boolProp>
          <stringProp name="LoopController.loops">10</stringProp>
        </elementProp>
        <stringProp name="ThreadGroup.num_threads">50</stringProp>
        <stringProp name="ThreadGroup.ramp_time">300</stringProp>
        <boolProp name="ThreadGroup.scheduler">true</boolProp>
        <stringProp name="ThreadGroup.duration">1800</stringProp>
      </ThreadGroup>
      <hashTree>
        <!-- HTTP Request Defaults -->
        <ConfigTestElement guiclass="HttpDefaultsGui" testclass="ConfigTestElement" testname="HTTP Request Defaults">
          <elementProp name="HTTPsampler.Arguments" elementType="Arguments">
            <collectionProp name="Arguments.arguments"/>
          </elementProp>
          <stringProp name="HTTPSampler.domain">${base_url}</stringProp>
          <stringProp name="HTTPSampler.protocol">http</stringProp>
          <stringProp name="HTTPSampler.contentEncoding">UTF-8</stringProp>
        </ConfigTestElement>
        <hashTree/>
        
        <!-- Cookie Manager -->
        <CookieManager guiclass="CookiePanel" testclass="CookieManager" testname="HTTP Cookie Manager">
          <collectionProp name="CookieManager.cookies"/>
          <boolProp name="CookieManager.clearEachIteration">false</boolProp>
          <boolProp name="CookieManager.controlledByThreadGroup">false</boolProp>
        </CookieManager>
        <hashTree/>
        
        <!-- Authentication -->
        <HTTPSamplerProxy guiclass="HttpTestSampleGui" testclass="HTTPSamplerProxy" testname="Login">
          <elementProp name="HTTPsampler.Arguments" elementType="Arguments">
            <collectionProp name="Arguments.arguments">
              <elementProp name="" elementType="HTTPArgument">
                <boolProp name="HTTPArgument.always_encode">false</boolProp>
                <stringProp name="Argument.value">{"email": "test@example.com", "password": "password123"}</stringProp>
                <stringProp name="Argument.metadata">=</stringProp>
              </elementProp>
            </collectionProp>
          </elementProp>
          <stringProp name="HTTPSampler.path">/api/auth/login</stringProp>
          <stringProp name="HTTPSampler.method">POST</stringProp>
          <boolProp name="HTTPSampler.use_keepalive">true</boolProp>
          <boolProp name="HTTPSampler.postBodyRaw">true</boolProp>
          <elementProp name="HTTPsampler.header_manager" elementType="HeaderManager">
            <collectionProp name="HeaderManager.headers">
              <elementProp name="" elementType="Header">
                <stringProp name="Header.name">Content-Type</stringProp>
                <stringProp name="Header.value">application/json</stringProp>
              </elementProp>
            </collectionProp>
          </elementProp>
        </HTTPSamplerProxy>
        <hashTree>
          <!-- Extract JWT Token -->
          <JSONPostProcessor guiclass="JSONPostProcessorGui" testclass="JSONPostProcessor" testname="Extract Token">
            <stringProp name="JSONPostProcessor.referenceNames">auth_token</stringProp>
            <stringProp name="JSONPostProcessor.jsonPathExprs">$.token</stringProp>
            <stringProp name="JSONPostProcessor.match_numbers">1</stringProp>
          </JSONPostProcessor>
          <hashTree/>
        </hashTree>
      </hashTree>
    </hashTree>
  </hashTree>
</jmeterTestPlan>

Artillery.js Load Testing

# artillery-config.yml
config:
  target: 'http://localhost:3000'
  phases:
    - duration: 60
      arrivalRate: 5
      name: 'Warm up phase'
    - duration: 300
      arrivalRate: 10
      rampTo: 50
      name: 'Load test phase'
    - duration: 120
      arrivalRate: 50
      name: 'Sustained load phase'
    - duration: 60
      arrivalRate: 100
      name: 'Stress test phase'
  
  processor: './test-processor.js'
  
  variables:
    users:
      - ['user1@test.com', 'password123']
      - ['user2@test.com', 'password123'] 
      - ['user3@test.com', 'password123']

scenarios:
  - name: 'User Journey - Browse and Purchase'
    weight: 60
    flow:
      - post:
          url: '/api/auth/login'
          json:
            email: '{{ $randomItem(users)[0] }}'
            password: '{{ $randomItem(users)[1] }}'
          capture:
            - json: '$.token'
              as: 'authToken'
      
      - get:
          url: '/api/products'
          headers:
            Authorization: 'Bearer {{ authToken }}'
          capture:
            - json: '$.data[0].id'
              as: 'productId'
      
      - get:
          url: '/api/products/{{ productId }}'
          headers:
            Authorization: 'Bearer {{ authToken }}'
      
      - post:
          url: '/api/orders'
          headers:
            Authorization: 'Bearer {{ authToken }}'
          json: {}
          capture:
            - json: '$.id'
              as: 'orderId'
      
      - post:
          url: '/api/orders/{{ orderId }}/items'
          headers:
            Authorization: 'Bearer {{ authToken }}'
          json:
            productId: '{{ productId }}'
            quantity: '{{ $randomInt(1, 5) }}'
      
      - think: 2
      
      - post:
          url: '/api/orders/{{ orderId }}/confirm'
          headers:
            Authorization: 'Bearer {{ authToken }}'
          json:
            paymentMethod: 'credit_card'

  - name: 'Admin Dashboard'
    weight: 20
    flow:
      - post:
          url: '/api/auth/login'
          json:
            email: 'admin@test.com'
            password: 'admin123'
          capture:
            - json: '$.token'
              as: 'authToken'
      
      - get:
          url: '/api/admin/dashboard'
          headers:
            Authorization: 'Bearer {{ authToken }}'
      
      - get:
          url: '/api/admin/analytics'
          headers:
            Authorization: 'Bearer {{ authToken }}'

  - name: 'API Health Checks'
    weight: 20
    flow:
      - get:
          url: '/health'
      - get:
          url: '/api/health'

Performance Monitoring Setup

// monitoring/performance-monitor.js
const prometheus = require('prom-client');
const express = require('express');

class PerformanceMonitor {
  constructor() {
    // Create metrics registry
    this.register = new prometheus.Registry();
    
    // Add default metrics
    prometheus.collectDefaultMetrics({ register: this.register });
    
    // Custom metrics
    this.httpRequestDuration = new prometheus.Histogram({
      name: 'http_request_duration_seconds',
      help: 'Duration of HTTP requests in seconds',
      labelNames: ['method', 'route', 'status_code'],
      buckets: [0.1, 0.3, 0.5, 0.7, 1, 3, 5, 7, 10],
      registers: [this.register]
    });
    
    this.httpRequestTotal = new prometheus.Counter({
      name: 'http_requests_total',
      help: 'Total number of HTTP requests',
      labelNames: ['method', 'route', 'status_code'],
      registers: [this.register]
    });
    
    this.activeConnections = new prometheus.Gauge({
      name: 'active_connections',
      help: 'Number of active connections',
      registers: [this.register]
    });
    
    this.databaseQueryDuration = new prometheus.Histogram({
      name: 'database_query_duration_seconds',
      help: 'Duration of database queries in seconds',
      labelNames: ['operation', 'table'],
      buckets: [0.01, 0.05, 0.1, 0.3, 0.5, 1, 3, 5],
      registers: [this.register]
    });
    
    this.memoryUsage = new prometheus.Gauge({
      name: 'memory_usage_bytes',
      help: 'Memory usage in bytes',
      labelNames: ['type'],
      registers: [this.register]
    });
    
    // Start memory monitoring
    this.startMemoryMonitoring();
  }
  
  // Express middleware for HTTP metrics
  getHttpMetricsMiddleware() {
    return (req, res, next) => {
      const startTime = Date.now();
      
      res.on('finish', () => {
        const duration = (Date.now() - startTime) / 1000;
        const route = req.route?.path || req.path || 'unknown';
        
        this.httpRequestDuration
          .labels(req.method, route, res.statusCode.toString())
          .observe(duration);
        
        this.httpRequestTotal
          .labels(req.method, route, res.statusCode.toString())
          .inc();
      });
      
      next();
    };
  }
  
  // Database query metrics
  recordDatabaseQuery(operation, table, duration) {
    this.databaseQueryDuration
      .labels(operation, table)
      .observe(duration / 1000); // Convert to seconds
  }
  
  // Connection metrics
  incrementActiveConnections() {
    this.activeConnections.inc();
  }
  
  decrementActiveConnections() {
    this.activeConnections.dec();
  }
  
  // Memory monitoring
  startMemoryMonitoring() {
    setInterval(() => {
      const memUsage = process.memoryUsage();
      
      this.memoryUsage.labels('rss').set(memUsage.rss);
      this.memoryUsage.labels('heapTotal').set(memUsage.heapTotal);
      this.memoryUsage.labels('heapUsed').set(memUsage.heapUsed);
      this.memoryUsage.labels('external').set(memUsage.external);
    }, 5000); // Every 5 seconds
  }
  
  // Metrics endpoint
  getMetricsHandler() {
    return async (req, res) => {
      res.set('Content-Type', this.register.contentType);
      res.end(await this.register.metrics());
    };
  }
}

module.exports = PerformanceMonitor;

Performance Profiling

// profiling/cpu-profiler.js
const v8Profiler = require('v8-profiler-next');
const fs = require('fs');
const path = require('path');

class CPUProfiler {
  constructor() {
    this.profiles = new Map();
  }
  
  startProfiling(profileName = 'default') {
    console.log(`Starting CPU profiling: ${profileName}`);
    v8Profiler.startProfiling(profileName, true);
    this.profiles.set(profileName, Date.now());
  }
  
  stopProfiling(profileName = 'default', outputDir = './profiles') {
    const profile = v8Profiler.stopProfiling(profileName);
    const startTime = this.profiles.get(profileName);
    const duration = Date.now() - startTime;
    
    console.log(`CPU profiling completed: ${profileName} (${duration}ms)`);
    
    // Ensure output directory exists
    if (!fs.existsSync(outputDir)) {
      fs.mkdirSync(outputDir, { recursive: true });
    }
    
    // Save profile
    const fileName = `cpu-profile-${profileName}-${Date.now()}.cpuprofile`;
    const filePath = path.join(outputDir, fileName);
    
    profile.export((error, result) => {
      if (error) {
        console.error('Error exporting CPU profile:', error);
        return;
      }
      
      fs.writeFileSync(filePath, result);
      console.log(`CPU profile saved: ${filePath}`);
    });
    
    profile.delete();
    this.profiles.delete(profileName);
  }
  
  // Automatic profiling for slow requests
  getProfilingMiddleware(threshold = 1000) {
    return (req, res, next) => {
      const startTime = Date.now();
      const profileName = `request-${Date.now()}-${Math.random().toString(36).substr(2, 9)}`;
      
      res.on('finish', () => {
        const duration = Date.now() - startTime;
        
        if (duration > threshold) {
          console.log(`Slow request detected: ${req.method} ${req.path} (${duration}ms)`);
          // Profile similar requests in the future
          this.scheduleProfilingForRoute(req.method, req.path);
        }
      });
      
      next();
    };
  }
  
  scheduleProfilingForRoute(method, path) {
    // Implementation for scheduled profiling
    console.log(`Scheduling profiling for ${method} ${path}`);
  }
}

// Memory profiler
class MemoryProfiler {
  constructor() {
    this.snapshots = [];
  }
  
  takeHeapSnapshot(name = 'default') {
    const snapshot = v8Profiler.takeSnapshot(name);
    this.snapshots.push({ name, snapshot, timestamp: Date.now() });
    
    return snapshot;
  }
  
  saveHeapSnapshot(snapshot, outputDir = './profiles') {
    return new Promise((resolve, reject) => {
      // Ensure output directory exists
      if (!fs.existsSync(outputDir)) {
        fs.mkdirSync(outputDir, { recursive: true });
      }
      
      const fileName = `heap-snapshot-${Date.now()}.heapsnapshot`;
      const filePath = path.join(outputDir, fileName);
      const writeStream = fs.createWriteStream(filePath);
      
      snapshot.export()
        .pipe(writeStream)
        .on('finish', () => {
          console.log(`Heap snapshot saved: ${filePath}`);
          resolve(filePath);
        })
        .on('error', (error) => {
          console.error('Error saving heap snapshot:', error);
          reject(error);
        });
    });
  }
  
  compareSnapshots(snapshot1, snapshot2) {
    // Simplified comparison - in practice, you'd use more sophisticated tools
    console.log('Comparing heap snapshots...');
    
    const diff = {
      timestamp1: snapshot1.timestamp,
      timestamp2: snapshot2.timestamp,
      timeDiff: snapshot2.timestamp - snapshot1.timestamp,
      // Add more detailed comparison logic here
    };
    
    return diff;
  }
  
  // Automatic memory monitoring
  startMemoryMonitoring(interval = 30000) {
    setInterval(() => {
      const memUsage = process.memoryUsage();
      const heapUsedMB = Math.round(memUsage.heapUsed / 1024 / 1024);
      
      console.log(`Memory usage: ${heapUsedMB}MB heap used`);
      
      // Take snapshot if memory usage is high
      if (heapUsedMB > 500) { // 500MB threshold
        console.log('High memory usage detected, taking heap snapshot');
        const snapshot = this.takeHeapSnapshot(`high-memory-${Date.now()}`);
        this.saveHeapSnapshot(snapshot);
      }
    }, interval);
  }
}

module.exports = { CPUProfiler, MemoryProfiler };

Performance Testing CI/CD Integration

# .github/workflows/performance-test.yml
name: Performance Testing

on:
  schedule:
    - cron: '0 2 * * *'  # Daily at 2 AM
  workflow_dispatch:
    inputs:
      test_type:
        description: 'Type of performance test'
        required: true
        default: 'load'
        type: choice
        options:
        - load
        - stress
        - spike
        - endurance

jobs:
  performance-test:
    runs-on: ubuntu-latest
    
    steps:
      - name: Checkout code
        uses: actions/checkout@v3
      
      - name: Setup Node.js
        uses: actions/setup-node@v3
        with:
          node-version: '18'
          cache: 'npm'
      
      - name: Install dependencies
        run: npm ci
      
      - name: Build application
        run: npm run build
      
      - name: Start application
        run: |
          npm start &
          sleep 30  # Wait for app to start
      
      - name: Install K6
        run: |
          sudo apt-key adv --keyserver hkp://keyserver.ubuntu.com:80 --recv-keys C5AD17C747E3415A3642D57D77C6C491D6AC1D69
          echo "deb https://dl.k6.io/deb stable main" | sudo tee /etc/apt/sources.list.d/k6.list
          sudo apt-get update
          sudo apt-get install k6
      
      - name: Run Performance Tests
        run: |
          case "${{ github.event.inputs.test_type || 'load' }}" in
            load)
              k6 run --out json=results.json k6-tests/load-test-suite.js
              ;;
            stress)
              k6 run --out json=results.json k6-tests/stress-test.js
              ;;
            spike)
              k6 run --out json=results.json k6-tests/spike-test.js
              ;;
            endurance)
              k6 run --out json=results.json k6-tests/endurance-test.js
              ;;
          esac
        env:
          BASE_URL: http://localhost:3000
      
      - name: Generate Performance Report
        run: |
          node scripts/generate-performance-report.js results.json
      
      - name: Upload Results
        uses: actions/upload-artifact@v3
        with:
          name: performance-test-results
          path: |
            results.json
            performance-report.html
            performance-report.json
      
      - name: Performance Regression Check
        run: |
          node scripts/check-performance-regression.js results.json
      
      - name: Comment PR with Results
        if: github.event_name == 'pull_request'
        uses: actions/github-script@v6
        with:
          script: |
            const fs = require('fs');
            const report = JSON.parse(fs.readFileSync('performance-report.json', 'utf8'));
            
            const comment = `## Performance Test Results
            
            | Metric | Value | Threshold | Status |
            |--------|-------|-----------|--------|
            | Avg Response Time | ${report.avg_response_time}ms | <500ms | ${report.avg_response_time < 500 ? '✅' : '❌'} |
            | P95 Response Time | ${report.p95_response_time}ms | <2000ms | ${report.p95_response_time < 2000 ? '✅' : '❌'} |
            | Error Rate | ${report.error_rate}% | <1% | ${report.error_rate < 1 ? '✅' : '❌'} |
            | Throughput | ${report.throughput} RPS | >100 RPS | ${report.throughput > 100 ? '✅' : '❌'} |
            
            [View detailed report](${report.report_url})
            `;
            
            github.rest.issues.createComment({
              issue_number: context.issue.number,
              owner: context.repo.owner,
              repo: context.repo.repo,
              body: comment
            });

Performance Optimization Recommendations

// scripts/performance-analyzer.js
class PerformanceAnalyzer {
  constructor() {
    this.recommendations = [];
  }
  
  analyzeResults(testResults) {
    const analysis = {
      responseTimeAnalysis: this.analyzeResponseTime(testResults),
      throughputAnalysis: this.analyzeThroughput(testResults),
      errorAnalysis: this.analyzeErrors(testResults),
      resourceAnalysis: this.analyzeResourceUsage(testResults),
    };
    
    this.generateRecommendations(analysis);
    
    return {
      analysis,
      recommendations: this.recommendations,
    };
  }
  
  analyzeResponseTime(results) {
    const responseTimes = results.metrics.http_req_duration.values;
    
    return {
      average: responseTimes.avg,
      p50: responseTimes.p50,
      p90: responseTimes.p90,
      p95: responseTimes.p95,
      p99: responseTimes.p99,
      max: responseTimes.max,
      isWithinSLA: responseTimes.p95 < 2000, // 2 second SLA
    };
  }
  
  analyzeThroughput(results) {
    const requestRate = results.metrics.http_reqs.rate;
    
    return {
      requestsPerSecond: requestRate,
      totalRequests: results.metrics.http_reqs.count,
      isWithinTarget: requestRate > 100, // 100 RPS target
    };
  }
  
  analyzeErrors(results) {
    const errorRate = results.metrics.http_req_failed.rate * 100;
    
    return {
      errorRate,
      totalErrors: results.metrics.http_req_failed.count,
      isWithinTarget: errorRate < 1, // 1% error rate target
    };
  }
  
  analyzeResourceUsage(results) {
    // This would typically come from monitoring data
    return {
      cpuUsage: results.monitoring?.cpu_usage || 0,
      memoryUsage: results.monitoring?.memory_usage || 0,
      diskIO: results.monitoring?.disk_io || 0,
      networkIO: results.monitoring?.network_io || 0,
    };
  }
  
  generateRecommendations(analysis) {
    this.recommendations = [];
    
    // Response time recommendations
    if (!analysis.responseTimeAnalysis.isWithinSLA) {
      this.recommendations.push({
        category: 'Response Time',
        severity: 'High',
        issue: `P95 response time (${analysis.responseTimeAnalysis.p95}ms) exceeds SLA`,
        recommendations: [
          'Implement database query optimization',
          'Add caching layer (Redis/Memcached)',
          'Optimize API endpoint logic',
          'Consider implementing CDN for static assets',
        ],
      });
    }
    
    // Throughput recommendations
    if (!analysis.throughputAnalysis.isWithinTarget) {
      this.recommendations.push({
        category: 'Throughput',
        severity: 'Medium',
        issue: `Request rate (${analysis.throughputAnalysis.requestsPerSecond} RPS) below target`,
        recommendations: [
          'Scale horizontally by adding more server instances',
          'Implement connection pooling',
          'Optimize application startup time',
          'Review and optimize middleware chain',
        ],
      });
    }
    
    // Error rate recommendations
    if (!analysis.errorAnalysis.isWithinTarget) {
      this.recommendations.push({
        category: 'Reliability',
        severity: 'Critical',
        issue: `Error rate (${analysis.errorAnalysis.errorRate}%) exceeds threshold`,
        recommendations: [
          'Implement circuit breaker pattern',
          'Add retry logic with exponential backoff',
          'Improve error handling and logging',
          'Review database connection management',
        ],
      });
    }
    
    // Resource usage recommendations
    if (analysis.resourceAnalysis.cpuUsage > 80) {
      this.recommendations.push({
        category: 'Resource Usage',
        severity: 'High',
        issue: `High CPU usage (${analysis.resourceAnalysis.cpuUsage}%)`,
        recommendations: [
          'Profile application to identify CPU bottlenecks',
          'Implement async processing for heavy operations',
          'Consider CPU scaling or optimization',
          'Review algorithm complexity in hot paths',
        ],
      });
    }
    
    if (analysis.resourceAnalysis.memoryUsage > 80) {
      this.recommendations.push({
        category: 'Resource Usage',
        severity: 'High',
        issue: `High memory usage (${analysis.resourceAnalysis.memoryUsage}%)`,
        recommendations: [
          'Implement memory profiling to identify leaks',
          'Optimize data structures and caching',
          'Consider memory scaling',
          'Review garbage collection settings',
        ],
      });
    }
  }
}

module.exports = PerformanceAnalyzer;

This performance testing expert provides comprehensive strategies and implementations for identifying bottlenecks, measuring performance metrics, and optimizing application performance across different load scenarios.