CLOUDP-351614: Fix monitoring failure after disabling TLS on AppDB

When TLS is disabled on AppDB, the operator now correctly clears stale
TLS parameters (additionalParams) from the monitoring configuration.

Before this fix, stale TLS params like useSslForAllConnections and
sslClientCertificate would remain in the monitoring config after TLS
was disabled, causing monitoring agents to fail when trying to use
certificate files that are no longer valid for authentication.

Changes:
- deployment.go: Clear additionalParams when TLS is disabled
- appdbreplicaset_controller.go: Use correct TLS state for monitoring
- Added e2e test (e2e_om_appdb_tls_disable) that verifies:
  1. TLS params are present when TLS is enabled
  2. TLS params are cleared after TLS is disabled
  3. Monitoring continues to work after TLS disable

Author: nammn

.evergreen-tasks.yml

@@ -783,6 +783,11 @@ tasks:
     commands:
       - func: "e2e_test"
 
+  - name: e2e_om_appdb_tls_disable
+    tags: [ "patch-run" ]
+    commands:
+      - func: "e2e_test"
+
   - name: e2e_om_appdb_multi_change
     tags: [ "patch-run" ]
     commands:

.evergreen.yml

@@ -968,6 +968,7 @@ task_groups:
       - e2e_om_appdb_flags_and_config
       - e2e_om_appdb_upgrade
       - e2e_om_appdb_monitoring_tls
+      - e2e_om_appdb_tls_disable
       - e2e_om_ops_manager_backup
       - e2e_om_ops_manager_backup_light
       - e2e_om_ops_manager_backup_liveness_probe
@@ -1022,6 +1023,7 @@ task_groups:
       - e2e_om_appdb_flags_and_config
       - e2e_om_appdb_upgrade
       - e2e_om_appdb_monitoring_tls
+      - e2e_om_appdb_tls_disable
       - e2e_om_ops_manager_backup
       - e2e_om_ops_manager_backup_light
       - e2e_om_ops_manager_backup_liveness_probe
@@ -1076,6 +1078,7 @@ task_groups:
       - e2e_om_appdb_external_connectivity
       - e2e_om_appdb_flags_and_config
       - e2e_om_appdb_monitoring_tls
+      - e2e_om_appdb_tls_disable
       - e2e_om_appdb_multi_change
       - e2e_om_appdb_scale_up_down
       - e2e_om_appdb_upgrade
@@ -1120,6 +1123,7 @@ task_groups:
       - e2e_om_appdb_external_connectivity
       - e2e_om_appdb_flags_and_config
       - e2e_om_appdb_monitoring_tls
+      - e2e_om_appdb_tls_disable
       - e2e_om_appdb_multi_change
       - e2e_om_appdb_scale_up_down
       - e2e_om_appdb_upgrade
@@ -1439,7 +1443,21 @@ buildvariants:
     tags: [ "pr_patch", "staging", "e2e_test_suite", "static" ]
     run_on:
       - ubuntu2404-large
-    <<: *base_om8_dependency
+    depends_on:
+      - name: build_om_images
+        variant: build_om80_images
+      - name: build_operator_ubi
+        variant: init_test_run
+      - name: build_init_database_image_ubi
+        variant: init_test_run
+      - name: build_test_image
+        variant: init_test_run
+      - name: build_init_appdb_images_ubi
+        variant: init_test_run
+      - name: build_init_om_images_ubi
+        variant: init_test_run
+      - name: publish_helm_chart
+        variant: init_test_run
     tasks:
       - name: e2e_static_ops_manager_kind_only_task_group
       - name: e2e_static_ops_manager_kind_6_0_only_task_group

changelog/20251216_fix_tls_monitoring.md

@@ -0,0 +1,6 @@
+---
+kind: fix
+date: 2025-12-16
+---
+
+* Fixed an issue where monitoring agents would fail after disabling TLS on a MongoDB deployment.

controllers/om/deployment.go

@@ -317,6 +317,10 @@ func (d Deployment) AddMonitoring(log *zap.SugaredLogger, tls bool, caFilePath s
 			}
 
 			monitoringVersion["additionalParams"] = additionalParams
+		} else {
+			// Clear TLS params when TLS is disabled to prevent monitoring from
+			// trying to use certificate files that no longer exist
+			delete(monitoringVersion, "additionalParams")
 		}
 
 	}

controllers/om/deployment_test.go

@@ -528,10 +528,39 @@ func TestAddMonitoringTls(t *testing.T) {
 	assert.Equal(t, expectedMonitoringVersions, d.getMonitoringVersions())
 
 	// adding again - nothing changes
-	d.AddMonitoring(zap.S(), false, util.CAFilePathInContainer)
+	d.AddMonitoring(zap.S(), true, util.CAFilePathInContainer)
 	assert.Equal(t, expectedMonitoringVersions, d.getMonitoringVersions())
 }
 
+func TestAddMonitoringTLSDisable(t *testing.T) {
+	d := NewDeployment()
+
+	rs0 := buildRsByProcesses("my-rs", createReplicaSetProcessesCount(3, "my-rs"))
+	d.MergeReplicaSet(rs0, nil, nil, zap.S())
+	d.AddMonitoring(zap.S(), true, util.CAFilePathInContainer)
+
+	// verify TLS is present in additionalParams
+	expectedAdditionalParams := map[string]string{
+		"useSslForAllConnections":      "true",
+		"sslTrustedServerCertificates": util.CAFilePathInContainer,
+	}
+	expectedMonitoringVersionsWithTls := []interface{}{
+		map[string]interface{}{"hostname": "my-rs-0.some.host", "name": MonitoringAgentDefaultVersion, "additionalParams": expectedAdditionalParams},
+		map[string]interface{}{"hostname": "my-rs-1.some.host", "name": MonitoringAgentDefaultVersion, "additionalParams": expectedAdditionalParams},
+		map[string]interface{}{"hostname": "my-rs-2.some.host", "name": MonitoringAgentDefaultVersion, "additionalParams": expectedAdditionalParams},
+	}
+	assert.Equal(t, expectedMonitoringVersionsWithTls, d.getMonitoringVersions())
+
+	// disabling TLS should clear additionalParams (CLOUDP-351614)
+	d.AddMonitoring(zap.S(), false, util.CAFilePathInContainer)
+	expectedMonitoringVersionsWithoutTls := []interface{}{
+		map[string]interface{}{"hostname": "my-rs-0.some.host", "name": MonitoringAgentDefaultVersion},
+		map[string]interface{}{"hostname": "my-rs-1.some.host", "name": MonitoringAgentDefaultVersion},
+		map[string]interface{}{"hostname": "my-rs-2.some.host", "name": MonitoringAgentDefaultVersion},
+	}
+	assert.Equal(t, expectedMonitoringVersionsWithoutTls, d.getMonitoringVersions())
+}
+
 func TestAddBackup(t *testing.T) {
 	d := NewDeployment()
 

controllers/operator/appdbreplicaset_controller.go

@@ -1430,9 +1430,14 @@ func addMonitoring(ac *automationconfig.AutomationConfig, log *zap.SugaredLogger
 	monitoringVersions := ac.MonitoringVersions
 	for _, p := range ac.Processes {
 		found := false
-		for _, m := range monitoringVersions {
+		for i, m := range monitoringVersions {
 			if m.Hostname == p.HostName {
 				found = true
+				if !tls {
+					// Clear TLS params when TLS is disabled to prevent monitoring from
+					// trying to use certificate files that no longer exist
+					monitoringVersions[i].AdditionalParams = nil
+				}
 				break
 			}
 		}

docker/mongodb-kubernetes-tests/tests/opsmanager/fixtures/om_appdb_tls_disable.yaml

@@ -0,0 +1,35 @@
+apiVersion: mongodb.com/v1
+kind: MongoDBOpsManager
+metadata:
+  name: om-tls-disable-test
+spec:
+  replicas: 1
+  version: 4.4.1
+  adminCredentials: ops-manager-admin-secret
+  security:
+    tls:
+      secretRef:
+        name: certs-for-ops-manager
+      ca: issuer-ca
+  backup:
+    enabled: false
+  applicationDatabase:
+    members: 3
+    version: 5.0.14-ent
+    security:
+      certsSecretPrefix: appdb
+      tls:
+        ca: issuer-ca
+
+  # adding this just to avoid wizard when opening OM UI
+  configuration:
+    automation.versions.source: mongodb
+    mms.adminEmailAddr: [email protected]
+    mms.fromEmailAddr: [email protected]
+    mms.ignoreInitialUiSetup: "true"
+    mms.mail.hostname: email-smtp.us-east-1.amazonaws.com
+    mms.mail.port: "465"
+    mms.mail.ssl: "true"
+    mms.mail.transport: smtp
+    mms.minimumTLSVersion: TLSv1.2
+    mms.replyToEmailAddr: [email protected]

docker/mongodb-kubernetes-tests/tests/opsmanager/withMonitoredAppDB/om_appdb_tls_disable.py

@@ -0,0 +1,179 @@
+"""
+CLOUDP-351614: Test that TLS can be disabled on AppDB without breaking monitoring.
+
+This is a dedicated test file to verify that when TLS is disabled on AppDB,
+the operator correctly clears stale TLS params from the monitoring configuration.
+"""
+from typing import Optional
+
+from kubetester import create_or_update_secret, try_load
+from kubetester.certs import create_ops_manager_tls_certs
+from kubetester.kubetester import fixture as yaml_fixture
+from kubetester.opsmanager import MongoDBOpsManager
+from kubetester.phase import Phase
+from pytest import fixture, mark
+from tests.common.cert.cert_issuer import create_appdb_certs
+from tests.conftest import is_multi_cluster
+from tests.opsmanager.withMonitoredAppDB.conftest import enable_multi_cluster_deployment
+
+
+def get_monitoring_tls_params(ops_manager: MongoDBOpsManager) -> dict:
+    """
+    Extract TLS-related additionalParams from monitoring config.
+
+    Returns a dict with the additionalParams for all monitoring agents,
+    keyed by hostname. An empty dict means no TLS params are present.
+    """
+    ac = ops_manager.get_automation_config_tester()
+    monitoring_versions = ac.automation_config.get("monitoringVersions", [])
+
+    tls_params_by_host = {}
+    for mv in monitoring_versions:
+        hostname = mv.get("hostname", "unknown")
+        additional_params = mv.get("additionalParams", {})
+        if additional_params:
+            tls_params_by_host[hostname] = additional_params
+
+    return tls_params_by_host
+
+OM_NAME = "om-tls-disable-test"
+APPDB_NAME = f"{OM_NAME}-db"
+
+
+@fixture(scope="module")
+def ops_manager_certs(namespace: str, issuer: str):
+    return create_ops_manager_tls_certs(issuer, namespace, OM_NAME)
+
+
+@fixture(scope="module")
+def appdb_certs(namespace: str, issuer: str):
+    return create_appdb_certs(namespace, issuer, APPDB_NAME)
+
+
+@fixture(scope="module")
+@mark.usefixtures("appdb_certs", "ops_manager_certs", "issuer_ca_configmap")
+def ops_manager(
+    namespace: str,
+    issuer_ca_configmap: str,
+    appdb_certs: str,
+    ops_manager_certs: str,
+    custom_version: Optional[str],
+    custom_appdb_version: str,
+) -> MongoDBOpsManager:
+    """Create an Ops Manager with TLS-enabled AppDB for testing TLS disable."""
+    om = MongoDBOpsManager.from_yaml(yaml_fixture("om_appdb_tls_disable.yaml"), namespace=namespace)
+    om.set_version(custom_version)
+    om.set_appdb_version(custom_appdb_version)
+
+    if try_load(om):
+        return om
+
+    if is_multi_cluster():
+        enable_multi_cluster_deployment(om)
+
+    om.update()
+    return om
+
+
+@mark.e2e_om_appdb_tls_disable
+def test_om_created_with_tls(ops_manager: MongoDBOpsManager):
+    """Verify OM and AppDB are running with TLS enabled."""
+    ops_manager.om_status().assert_reaches_phase(Phase.Running, timeout=900)
+    ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=600)
+
+
+@mark.e2e_om_appdb_tls_disable
+def test_appdb_monitoring_works_with_tls(ops_manager: MongoDBOpsManager):
+    """Verify monitoring works with TLS enabled before we disable it."""
+    ops_manager.assert_appdb_monitoring_group_was_created()
+    ops_manager.assert_monitoring_data_exists(timeout=600, all_hosts=False)
+
+
+@mark.e2e_om_appdb_tls_disable
+def test_monitoring_config_has_tls_params_when_tls_enabled(ops_manager: MongoDBOpsManager):
+    """
+    CLOUDP-351614: Verify that monitoring config contains TLS params when TLS is enabled.
+
+    When TLS is enabled on AppDB, the monitoring agents should be configured with
+    TLS parameters in additionalParams, including:
+    - useSslForAllConnections: "true"
+    - sslTrustedServerCertificates: path to CA file
+    - sslClientCertificate: path to client certificate (for x509 auth)
+    """
+    tls_params = get_monitoring_tls_params(ops_manager)
+
+    # Monitoring agents should have TLS params configured
+    assert len(tls_params) > 0, "Expected TLS params in monitoring config when TLS is enabled"
+
+    # Verify TLS params contain expected keys
+    for hostname, params in tls_params.items():
+        assert params.get("useSslForAllConnections") == "true", (
+            f"Expected useSslForAllConnections=true for {hostname}, got {params}"
+        )
+        assert "sslTrustedServerCertificates" in params, (
+            f"Expected sslTrustedServerCertificates in params for {hostname}"
+        )
+
+
+@mark.e2e_om_appdb_tls_disable
+def test_disable_tls_on_appdb(ops_manager: MongoDBOpsManager):
+    """
+    CLOUDP-351614: Disable TLS on AppDB and verify the operator correctly handles
+    the transition without leaving stale TLS params in monitoring config.
+
+    This test disables TLS by setting tls.enabled = False. The operator handles
+    the TLS mode transition: requireTLS -> preferTLS -> allowTLS -> disabled.
+
+    Note: We keep the certsSecretPrefix in place because:
+    1. The cert files are needed during the TLS transition
+    2. Removing certsSecretPrefix while TLS mode is still transitioning causes failures
+    3. The certs are harmless to keep after TLS is disabled (just unused)
+    """
+    ops_manager.load()
+
+    # Disable TLS mode (keeping certs for the transition)
+    # The operator will handle the TLS mode transition: requireTLS -> preferTLS -> allowTLS -> disabled
+    ops_manager["spec"]["applicationDatabase"]["security"]["tls"]["enabled"] = False
+    ops_manager.update()
+
+    # Wait for AppDB to reach Running state after TLS mode is fully disabled
+    # Use a longer timeout as the transition goes through multiple TLS modes
+    ops_manager.appdb_status().assert_reaches_phase(Phase.Running, timeout=1800)
+
+
+@mark.e2e_om_appdb_tls_disable
+def test_monitoring_config_tls_params_cleared_after_tls_disabled(ops_manager: MongoDBOpsManager):
+    """
+    CLOUDP-351614: Verify that TLS params are CLEARED from monitoring config after TLS is disabled.
+
+    THIS IS THE KEY TEST FOR THE BUG FIX:
+    Before the fix in CLOUDP-351614, the operator would leave stale TLS params
+    (useSslForAllConnections, sslClientCertificate, etc.) in the monitoring config
+    even after TLS was disabled. This caused monitoring agents to fail because they
+    would try to use TLS certificate files that are no longer valid for authentication.
+
+    After the fix, the operator correctly clears these params by calling:
+        delete(monitoringVersion, "additionalParams")
+    """
+    tls_params = get_monitoring_tls_params(ops_manager)
+
+    # After TLS is disabled, monitoring config should NOT have TLS params
+    # This assertion would have FAILED before the fix because stale TLS params remained
+    assert len(tls_params) == 0, (
+        f"CLOUDP-351614 BUG: TLS params should be cleared from monitoring config after "
+        f"TLS is disabled, but found stale params: {tls_params}"
+    )
+
+
+@mark.e2e_om_appdb_tls_disable
+def test_monitoring_works_after_tls_disable(ops_manager: MongoDBOpsManager):
+    """
+    CLOUDP-351614: Verify monitoring data can still be collected after TLS disable.
+
+    After TLS is disabled, monitoring agents switch from x509 to SCRAM authentication.
+    This test verifies that:
+    1. The operator correctly cleared stale TLS params from monitoring config
+    2. Monitoring agents can reconnect and collect data
+    """
+    # Use a longer timeout as monitoring agents need time to reconnect with SCRAM auth
+    ops_manager.assert_monitoring_data_exists(timeout=1200, all_hosts=False)