optimise prometheus scraping workflow

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporter.java

@@ -18,7 +18,23 @@
 
 public interface PrometheusExporter {
 
+    /**
+     * Update the Prometheus metrics in text format.
+     *
+     * NOTE: capacity data is refreshed independently by {@code AlertManagerImpl}'s own
+     * periodic {@code CapacityChecker} timer. Do NOT force a synchronous
+     * {@code recalculateCapacity()} call here: it spins up a fresh thread pool per host
+     * and per storage pool across ALL zones on every single scrape, so with Z zones a
+     * single Prometheus scrape triggered Z redundant full recalculations. That extra,
+     * uncoordinated load compounds over time and can lead to {@code scrape_duration_seconds}
+     * climbing until a management-server restart.
+     *
+     * @see PrometheusExporterImpl#updateMetrics()
+     */
     void updateMetrics();
 
+    /**
+     * @return the latest Prometheus metrics refreshed by {@link #updateMetrics()}.
+     */
     String getMetrics();
 }

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterImpl.java

@@ -23,6 +23,7 @@
 import java.util.HashSet;
 import java.util.List;
 import java.util.Map;
+import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
 
 import javax.inject.Inject;
@@ -32,7 +33,6 @@
 import org.apache.cloudstack.storage.datastore.db.ImageStoreDao;
 import org.apache.commons.lang3.StringUtils;
 
-import com.cloud.alert.AlertManager;
 import com.cloud.api.ApiDBUtils;
 import com.cloud.api.query.dao.DomainJoinDao;
 import com.cloud.api.query.dao.StoragePoolJoinDao;
@@ -102,6 +102,7 @@
     }
 
     private static List<Item> metricsItems = new ArrayList<>();
+    private volatile long lastMetricsUpdateTime = 0L;
 
     @Inject
     private DataCenterDao dcDao;
@@ -126,8 +127,6 @@
     @Inject
     private DomainJoinDao domainDao;
     @Inject
-    private AlertManager alertManager;
-    @Inject
     DedicatedResourceDao _dedicatedDao;
     @Inject
     private AccountDao _accountDao;
@@ -491,13 +490,20 @@
     }
 
     @Override
-    public void updateMetrics() {
+    public synchronized void updateMetrics() {
+        final long minIntervalMs = TimeUnit.SECONDS.toMillis(PrometheusExporterServer.PrometheusExporterMinRefreshInterval.value());
+        final long now = System.currentTimeMillis();
+        if (now - lastMetricsUpdateTime < minIntervalMs) {
+            logger.debug("Skipping metrics recomputation, last update was " + (now - lastMetricsUpdateTime) + "ms ago (min interval: " + minIntervalMs + "ms)");
+            return;
+        }
+
+        final long startNanos = System.nanoTime();
         final List<Item> latestMetricsItems = new ArrayList<Item>();
         try {
             for (final DataCenterVO dc : dcDao.listAll()) {
                 final String zoneName = dc.getName();
                 final String zoneUuid = dc.getUuid();
-                alertManager.recalculateCapacity();
                 addHostMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
                 addVMMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
                 addVolumeMetrics(latestMetricsItems, dc.getId(), zoneName, zoneUuid);
@@ -512,8 +518,12 @@
             addDomainResourceCount(latestMetricsItems);
         } catch (Exception e) {
             logger.warn("Getting metrics failed ", e);
+        } finally {
+            final long elapsedMs = TimeUnit.NANOSECONDS.toMillis(System.nanoTime() - startNanos);
+            logger.info("Prometheus metrics update completed in " + elapsedMs + " ms");
         }
         metricsItems = latestMetricsItems;
+        lastMetricsUpdateTime = System.currentTimeMillis();
     }
 
     @Override

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServer.java

@@ -33,4 +33,7 @@ public interface PrometheusExporterServer extends Manager {
 
     ConfigKey<Integer> PrometheusExporterOfferingCountLimit = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.offering.output.limit", "-1",
             "Limit the number of output for cloudstack_vms_total_by_size to the provided value. -1 for unlimited output.", true);
+
+    ConfigKey<Integer> PrometheusExporterMinRefreshInterval = new ConfigKey<>("Advanced", Integer.class, "prometheus.exporter.metrics.min.refresh.interval", "5",
+            "Minimum interval in seconds between metrics recomputations. Scrapes arriving faster than this interval reuse the previously computed metrics.", true, EnablePrometheusExporter.key());
 }

plugins/integrations/prometheus/src/main/java/org/apache/cloudstack/metrics/PrometheusExporterServerImpl.java

@@ -29,10 +29,13 @@
 import java.net.InetSocketAddress;
 import java.nio.charset.StandardCharsets;
 import java.util.Arrays;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
 
 public class PrometheusExporterServerImpl extends ManagerBase implements PrometheusExporterServer, Configurable {
 
     private static HttpServer httpServer;
+    private ExecutorService httpExecutor;
 
     @Inject
     private PrometheusExporter prometheusExporter;
@@ -79,6 +82,8 @@ public boolean start() {
         if (EnablePrometheusExporter.value()) {
             try {
                 httpServer = HttpServer.create(new InetSocketAddress(PrometheusExporterServerPort.value()), 0);
+                httpExecutor = Executors.newFixedThreadPool(2);
+                httpServer.setExecutor(httpExecutor);
                 httpServer.createContext("/metrics", new ExporterHandler(prometheusExporter));
                 httpServer.createContext("/", new HttpHandler() {
                     @Override
@@ -105,9 +110,15 @@ public void handle(HttpExchange httpExchange) throws IOException {
     @Override
     public boolean stop() {
         if (httpServer != null) {
+            httpServer.setExecutor(null);
             httpServer.stop(0);
             logger.debug("Stopped Prometheus exporter http server");
         }
+        if (httpExecutor != null) {
+            httpExecutor.shutdownNow();
+            logger.debug("Shut down Prometheus exporter http executor");
+        }
+        httpExecutor = null;
         return true;
     }
 
@@ -122,7 +133,8 @@ public ConfigKey<?>[] getConfigKeys() {
                 EnablePrometheusExporter,
                 PrometheusExporterServerPort,
                 PrometheusExporterAllowedAddresses,
-                PrometheusExporterOfferingCountLimit
+                PrometheusExporterOfferingCountLimit,
+                PrometheusExporterMinRefreshInterval
         };
     }
 }

plugins/integrations/prometheus/src/test/java/org/apache/cloudstack/metrics/PrometheusExporterImplTest.java

@@ -18,6 +18,15 @@
 
 import static org.junit.Assert.assertEquals;
 import static org.junit.Assert.assertTrue;
+import static org.mockito.Mockito.mock;
+import static org.mockito.Mockito.times;
+import static org.mockito.Mockito.verify;
+import static org.mockito.Mockito.when;
+
+import java.lang.reflect.Field;
+import java.util.Collections;
+
+import com.cloud.dc.dao.DataCenterDao;
 
 import org.junit.Test;
 
@@ -105,4 +114,66 @@ public void testItemHostCertExpiryContainsTimestampValue() {
         assertTrue("Metric should contain correct timestamp value",
                 metricsString.endsWith(" " + CERT_EXPIRY_EPOCH));
     }
+
+    /**
+     * Two rapid calls to updateMetrics() within the min refresh interval
+     * should result in only one actual recomputation (one call to dcDao.listAll()).
+     */
+    @Test
+    public void testUpdateMetricsTTLGuardSkipsSecondCall() throws Exception {
+        PrometheusExporterImpl exporter = new PrometheusExporterImpl();
+
+        DataCenterDao mockDcDao = mock(DataCenterDao.class);
+        when(mockDcDao.listAll()).thenReturn(Collections.emptyList());
+        setField(exporter, "dcDao", mockDcDao);
+
+        // First call should trigger recomputation
+        exporter.updateMetrics();
+        // Second immediate call should be skipped by the TTL guard
+        exporter.updateMetrics();
+
+        verify(mockDcDao, times(1)).listAll();
+    }
+
+    /**
+     * After the min refresh interval has elapsed, updateMetrics() should
+     * trigger a fresh recomputation.
+     */
+    @Test
+    public void testUpdateMetricsTTLGuardAllowsAfterInterval() throws Exception {
+        PrometheusExporterImpl exporter = new PrometheusExporterImpl();
+
+        DataCenterDao mockDcDao = mock(DataCenterDao.class);
+        when(mockDcDao.listAll()).thenReturn(Collections.emptyList());
+        setField(exporter, "dcDao", mockDcDao);
+
+        // First call
+        exporter.updateMetrics();
+
+        // Simulate that the min interval has already elapsed by resetting lastMetricsUpdateTime
+        setField(exporter, "lastMetricsUpdateTime", 0L);
+
+        // Second call should now trigger recomputation
+        exporter.updateMetrics();
+
+        verify(mockDcDao, times(2)).listAll();
+    }
+
+    private static void setField(Object target, String fieldName, Object value) throws Exception {
+        Field field = null;
+        Class<?> clazz = target.getClass();
+        while (clazz != null) {
+            try {
+                field = clazz.getDeclaredField(fieldName);
+                break;
+            } catch (NoSuchFieldException e) {
+                clazz = clazz.getSuperclass();
+            }
+        }
+        if (field == null) {
+            throw new NoSuchFieldException(fieldName);
+        }
+        field.setAccessible(true);
+        field.set(target, value);
+    }
 }

server/src/main/java/com/cloud/alert/AlertManagerImpl.java

@@ -161,6 +161,8 @@ public class AlertManagerImpl extends ManagerBase implements AlertManager, Confi
 
     private final ExecutorService _executor;
 
+    private ExecutorService capacityExecutorService;
+
     protected SMTPMailSender mailSender;
     protected String[] recipients = null;
     protected String senderAddress = null;
@@ -249,6 +251,9 @@ public boolean start() {
     @Override
     public boolean stop() {
         _timer.cancel();
+        if (capacityExecutorService != null) {
+            capacityExecutorService.shutdown();
+        }
         return true;
     }
 
@@ -281,6 +286,24 @@ public void sendAlert(AlertType alertType, long dataCenterId, Long podId, String
         }
     }
 
+    /**
+     * Shared, long-lived pool for capacity recalculation, reused across every
+     * recalculateHostCapacities()/recalculateStorageCapacities() call instead of creating and
+     * tearing down a new thread pool per invocation. Repeatedly creating/shutting down pools was
+     * unnecessary overhead under frequent callers (e.g. the Prometheus exporter used to trigger a
+     * full recalculation on every scrape, see https://github.com/apache/cloudstack/issues/13586).
+     * Lazily created so this remains safe for callers that invoke the recalculate methods directly
+     * without going through configure()/start() (e.g. unit tests).
+     */
+    private synchronized ExecutorService getCapacityExecutorService() {
+        if (capacityExecutorService == null || capacityExecutorService.isShutdown()) {
+            capacityExecutorService = Executors.newFixedThreadPool(
+                    Math.max(1, CapacityManager.CapacityCalculateWorkers.value()),
+                    new NamedThreadFactory("Capacity-Calculator"));
+        }
+        return capacityExecutorService;
+    }
+
     /**
      * Recalculates the capacities of hosts, including CPU and RAM.
      */
@@ -290,10 +313,8 @@ protected void recalculateHostCapacities() {
             return;
         }
         ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
-        ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
-                Math.min(CapacityManager.CapacityCalculateWorkers.value(), hostIds.size())));
         for (Long hostId : hostIds) {
-            futures.put(hostId, executorService.submit(() -> {
+            futures.put(hostId, getCapacityExecutorService().submit(() -> {
                 final HostVO host = hostDao.findById(hostId);
                 _capacityMgr.updateCapacityForHost(host);
                 return null;
@@ -307,7 +328,6 @@ protected void recalculateHostCapacities() {
                         entry.getKey(), e.getMessage()), e);
             }
         }
-        executorService.shutdown();
     }
 
     protected void recalculateStorageCapacities() {
@@ -316,10 +336,8 @@ protected void recalculateStorageCapacities() {
             return;
         }
         ConcurrentHashMap<Long, Future<Void>> futures = new ConcurrentHashMap<>();
-        ExecutorService executorService = Executors.newFixedThreadPool(Math.max(1,
-                Math.min(CapacityManager.CapacityCalculateWorkers.value(), storagePoolIds.size())));
         for (Long poolId: storagePoolIds) {
-            futures.put(poolId, executorService.submit(() -> {
+            futures.put(poolId, getCapacityExecutorService().submit(() -> {
                 Transaction.execute(new TransactionCallbackNoReturn() {
                     @Override
                     public void doInTransactionWithoutResult(TransactionStatus status) {
@@ -343,7 +361,6 @@ public void doInTransactionWithoutResult(TransactionStatus status) {
                         entry.getKey(), e.getMessage()), e);
             }
         }
-        executorService.shutdown();
     }
 
     @Override