feat(gateway): safely rotate ACME credentials

docs/tutorials/gateway-service-setup.md

@@ -319,6 +319,20 @@ curl -sf -X POST "http://$ADMIN_ADDR/prpc/SetCertbotConfig" \
   }' && echo "Certbot config set (PRODUCTION)"
 ```
 
+The stored ACME account still belongs to the staging directory, so rotate the
+shared credentials. This registers a production account and re-pins every ZT
+domain's CAA records to it in one step (renewals refuse to run while the
+stored account and the configured ACME URL disagree):
+
+```bash
+curl -sf -X POST "http://$ADMIN_ADDR/prpc/RotateAcmeCredentials" \
+  -H "Content-Type: application/json" -d '{}' && echo "ACME account rotated"
+```
+
+> If the rotation reports that CAA re-pinning failed for some domains, the new
+> account is already published — rerun `SetCaa` until it succeeds instead of
+> rotating again (each rotation registers a new rate-limited ACME account).
+
 After switching the ACME URL, the renewal loop may report "does not need renewal" because the staging cert is still valid. Force a renewal for each ZT domain to get production certificates immediately:
 
 ```bash

dstack/gateway/rpc/proto/gateway_rpc.proto

@@ -164,6 +164,14 @@ message AcmeInfoResponse {
   string account_attestation = 5;
 }
 
+// Result of replacing the shared ACME account credentials.
+message RotateAcmeCredentialsResponse {
+  // URI of the newly-created ACME account. The private credentials are never returned.
+  string account_uri = 1;
+  // Number of ZT domains whose CAA records were updated for the new account.
+  uint32 domains_updated = 2;
+}
+
 // Get HostInfo for associated instance id.
 message GetInfoRequest {
   string id = 1;
@@ -466,6 +474,15 @@ service Admin {
   rpc GetCertbotConfig(google.protobuf.Empty) returns (CertbotConfigResponse) {}
   // Set global certbot configuration (includes ACME URL)
   rpc SetCertbotConfig(SetCertbotConfigRequest) returns (google.protobuf.Empty) {}
+  // Create a new ACME account, publish the shared credentials, and re-pin
+  // every ZT-domain CAA record to the new account. If CAA re-pinning fails for
+  // some domains, the new credentials are already published; rerun SetCaa
+  // until it succeeds instead of retrying the rotation (each rotation
+  // registers a new rate-limited ACME account). Rotation is serialized across
+  // nodes by a best-effort lock (WaveKV has no compare-and-swap), so still
+  // avoid rotating from multiple gateways concurrently. This re-pins issuance
+  // to the new account; it does not deactivate the old ACME account at the CA.
+  rpc RotateAcmeCredentials(google.protobuf.Empty) returns (RotateAcmeCredentialsResponse) {}
 
   // ==================== Per-Instance Port Policy Override ====================
   // Set an admin override for an instance's port policy. Takes precedence

dstack/gateway/src/admin_service.rs

@@ -18,10 +18,10 @@ use dstack_gateway_rpc::{
     ListCertAttestationsResponse, ListDnsCredentialsResponse, ListZtDomainsResponse,
     NodeStatusEntry, PeerSyncStatus as ProtoPeerSyncStatus, PortAttrs as RpcPortAttrs,
     PortPolicy as RpcPortPolicy, RenewCertResponse, RenewZtDomainCertRequest,
-    RenewZtDomainCertResponse, SetCertbotConfigRequest, SetDefaultDnsCredentialRequest,
-    SetInstancePortPolicyRequest, SetNodeStatusRequest, SetNodeUrlRequest, StatusResponse,
-    StoreSyncStatus, UpdateDnsCredentialRequest, WaveKvStatusResponse, ZtDomainCertStatus,
-    ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo,
+    RenewZtDomainCertResponse, RotateAcmeCredentialsResponse, SetCertbotConfigRequest,
+    SetDefaultDnsCredentialRequest, SetInstancePortPolicyRequest, SetNodeStatusRequest,
+    SetNodeUrlRequest, StatusResponse, StoreSyncStatus, UpdateDnsCredentialRequest,
+    WaveKvStatusResponse, ZtDomainCertStatus, ZtDomainConfig as ProtoZtDomainConfig, ZtDomainInfo,
 };
 use ra_rpc::{CallContext, RpcCall};
 use tracing::info;
@@ -100,6 +100,14 @@ impl AdminRpc for AdminRpcHandler {
         self.state.reload_all_certs_from_kvstore()
     }
 
+    async fn rotate_acme_credentials(self) -> Result<RotateAcmeCredentialsResponse> {
+        let (account_uri, domains_updated) = self.state.rotate_acme_credentials().await?;
+        Ok(RotateAcmeCredentialsResponse {
+            account_uri,
+            domains_updated: domains_updated.try_into().unwrap_or(u32::MAX),
+        })
+    }
+
     async fn status(self) -> Result<StatusResponse> {
         self.status().await
     }