feat(warden): prometheus metrics

11 files changed, 265 insertions, 11 deletions

diff --git a/Cargo.lock b/Cargo.lock
index 9e0ba4c..3a95300 100644
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -18,6 +18,18 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "320119579fcad9c21884f5c4861d16174d0e06250625266f50fe6898340abefa"
 
 [[package]]
+name = "ahash"
+version = "0.8.12"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5a15f179cd60c4584b8a8c596927aadc462e27f2ca70c04e0071964a73ba7a75"
+dependencies = [
+ "cfg-if",
+ "once_cell",
+ "version_check",
+ "zerocopy",
+]
+
+[[package]]
 name = "aho-corasick"
 version = "1.1.3"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -355,6 +367,21 @@ dependencies = [
 ]
 
 [[package]]
+name = "crossbeam-epoch"
+version = "0.9.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5b82ac4a3c2ca9c3460964f020e1402edd5753411d7737aa39c3714ad1b5420e"
+dependencies = [
+ "crossbeam-utils",
+]
+
+[[package]]
+name = "crossbeam-utils"
+version = "0.8.21"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d0a5c400df2834b80a4c3327b3aad3a4c4cd4de0629063962b03235697506a28"
+
+[[package]]
 name = "crypto-common"
 version = "0.1.6"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -493,6 +520,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1"
 
 [[package]]
+name = "foldhash"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2"
+
+[[package]]
 name = "form_urlencoded"
 version = "1.2.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -632,6 +665,9 @@ name = "hashbrown"
 version = "0.15.5"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "9229cfe53dfd69f0609a49f65461bd93001ea1ef889cd5529dd176593f5338a1"
+dependencies = [
+ "foldhash",
+]
 
 [[package]]
 name = "heck"
@@ -1019,6 +1055,46 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "32a282da65faaf38286cf3be983213fcf1d2e2a58700e808f83f4ea9a4804bc0"
 
 [[package]]
+name = "metrics"
+version = "0.24.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "25dea7ac8057892855ec285c440160265225438c3c45072613c25a4b26e98ef5"
+dependencies = [
+ "ahash",
+ "portable-atomic",
+]
+
+[[package]]
+name = "metrics-exporter-prometheus"
+version = "0.17.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2b166dea96003ee2531cf14833efedced545751d800f03535801d833313f8c15"
+dependencies = [
+ "base64",
+ "indexmap",
+ "metrics",
+ "metrics-util",
+ "quanta",
+ "thiserror",
+]
+
+[[package]]
+name = "metrics-util"
+version = "0.20.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "fe8db7a05415d0f919ffb905afa37784f71901c9a773188876984b4f769ab986"
+dependencies = [
+ "crossbeam-epoch",
+ "crossbeam-utils",
+ "hashbrown",
+ "metrics",
+ "quanta",
+ "rand",
+ "rand_xoshiro",
+ "sketches-ddsketch",
+]
+
+[[package]]
 name = "mime"
 version = "0.3.17"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1242,6 +1318,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
 
 [[package]]
+name = "portable-atomic"
+version = "1.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f84267b20a16ea918e43c6a88433c2d54fa145c92a811b5b047ccbe153674483"
+
+[[package]]
 name = "potential_utf"
 version = "0.1.2"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1391,6 +1473,21 @@ dependencies = [
 ]
 
 [[package]]
+name = "quanta"
+version = "0.12.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3ab5a9d756f0d97bdc89019bd2e4ea098cf9cde50ee7564dde6b81ccc8f06c7"
+dependencies = [
+ "crossbeam-utils",
+ "libc",
+ "once_cell",
+ "raw-cpuid",
+ "wasi 0.11.1+wasi-snapshot-preview1",
+ "web-sys",
+ "winapi",
+]
+
+[[package]]
 name = "quinn"
 version = "0.11.8"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1490,6 +1587,24 @@ dependencies = [
 ]
 
 [[package]]
+name = "rand_xoshiro"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f703f4665700daf5512dcca5f43afa6af89f09db47fb56be587f80636bda2d41"
+dependencies = [
+ "rand_core",
+]
+
+[[package]]
+name = "raw-cpuid"
+version = "11.5.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c6df7ab838ed27997ba19a4664507e6f82b41fe6e20be42929332156e5e85146"
+dependencies = [
+ "bitflags",
+]
+
+[[package]]
 name = "regex"
 version = "1.11.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -1805,6 +1920,12 @@ source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "d66dc143e6b11c1eddc06d5c423cfc97062865baf299914ab64caa38182078fe"
 
 [[package]]
+name = "sketches-ddsketch"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "c1e9a774a6c28142ac54bb25d25562e6bcf957493a184f15ad4eebccb23e410a"
+
+[[package]]
 name = "slab"
 version = "0.4.10"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2230,9 +2351,12 @@ dependencies = [
  "http-body",
  "iri-string",
  "pin-project-lite",
+ "tokio",
  "tower",
  "tower-layer",
  "tower-service",
+ "tracing",
+ "uuid",
 ]
 
 [[package]]
@@ -2509,6 +2633,17 @@ dependencies = [
 ]
 
 [[package]]
+name = "uuid"
+version = "1.17.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3cf4199d1e5d15ddd86a694e4d0dffa9c323ce759fea589f00fef9d81cc1931d"
+dependencies = [
+ "getrandom 0.3.3",
+ "js-sys",
+ "wasm-bindgen",
+]
+
+[[package]]
 name = "valuable"
 version = "0.1.1"
 source = "registry+https://github.com/rust-lang/crates.io-index"
@@ -2547,12 +2682,15 @@ dependencies = [
  "axum",
  "clap",
  "config",
+ "metrics",
+ "metrics-exporter-prometheus",
  "serde",
  "serde_json",
  "stack-up",
  "time",
  "tokio",
  "tower",
+ "tower-http",
  "tracing",
  "utoipa",
  "utoipa-axum",
diff --git a/Cargo.toml b/Cargo.toml
index 5a4c570..ef6e297 100644
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -13,6 +13,8 @@ anyhow = "1.0.98"
 axum = "0.8.4"
 clap = "4.5.43"
 config = { version = "0.15.13", default-features = false }
+metrics = { version = "0.24.2", default-features = false }
+metrics-exporter-prometheus = { version = "0.17.2", default-features = false }
 prost = "0.14.1"
 serde = "1.0.219"
 serde_json = "1.0.142"
@@ -21,6 +23,7 @@ time = "0.3.41"
 tokio = "1.47.1"
 tonic = "0.14.0"
 tower = "0.5.2"
+tower-http = "0.6.6"
 tracing = "0.1.41"
 utoipa = "5.4.0"
 utoipa-axum = "0.2.0"
diff --git a/crates/warden/Cargo.toml b/crates/warden/Cargo.toml
index cbe4656..d66c15e 100644
--- a/crates/warden/Cargo.toml
+++ b/crates/warden/Cargo.toml
@@ -12,10 +12,17 @@ anyhow.workspace = true
 axum = { workspace = true, features = ["macros"] }
 clap = { workspace = true, features = ["derive"] }
 config = { workspace = true, features = ["convert-case", "toml"] }
+metrics.workspace = true
+metrics-exporter-prometheus.workspace = true
 serde = { workspace = true, features = ["derive"] }
 serde_json.workspace = true
 time.workspace = true
 tokio = { workspace = true, features = ["macros", "rt-multi-thread", "signal"] }
+tower-http = { workspace = true, features = [
+    "timeout",
+    "trace",
+    "request-id",
+] }
 tracing.workspace = true
 utoipa = { workspace = true, features = ["axum_extras"] }
 utoipa-axum.workspace = true
diff --git a/crates/warden/src/main.rs b/crates/warden/src/main.rs
index ab7c4cf..b649280 100644
--- a/crates/warden/src/main.rs
+++ b/crates/warden/src/main.rs
@@ -42,7 +42,7 @@ async fn main() -> Result<(), error::AppError> {
         .loki(&config.application, &config.monitoring)?
         .build(&config.monitoring);
 
-      tokio::spawn(tracing.loki_task);
+    tokio::spawn(tracing.loki_task);
 
     let state = AppState::create(&config).await?;
 
diff --git a/crates/warden/src/server.rs b/crates/warden/src/server.rs
index a1968bb..ce01fb8 100644
--- a/crates/warden/src/server.rs
+++ b/crates/warden/src/server.rs
@@ -1,3 +1,4 @@
+mod middleware;
 mod routes;
 
 use axum::Router;
@@ -9,10 +10,13 @@ use utoipa_redoc::Servable;
 #[cfg(feature = "scalar")]
 use utoipa_scalar::Servable as _;
 
-use crate::{server::routes::ApiDoc, state::AppHandle};
+use crate::{
+    server::routes::{ApiDoc, metrics::metrics_app},
+    state::AppHandle,
+};
 
 pub fn router(state: AppHandle) -> Router {
-    let (router, api) = OpenApiRouter::with_openapi(ApiDoc::openapi())
+    let (router, _api) = OpenApiRouter::with_openapi(ApiDoc::openapi())
         .routes(routes!(health_check))
         .nest("/api", routes::processor::router(state.clone()))
         .split_for_parts();
@@ -20,22 +24,22 @@ pub fn router(state: AppHandle) -> Router {
     #[cfg(feature = "swagger")]
     let router = router.merge(
         utoipa_swagger_ui::SwaggerUi::new("/swagger-ui")
-            .url("/api-docs/swaggerdoc.json", api.clone()),
+            .url("/api-docs/swaggerdoc.json", _api.clone()),
     );
 
     #[cfg(feature = "redoc")]
-    let router = router.merge(utoipa_redoc::Redoc::with_url("/redoc", api.clone()));
+    let router = router.merge(utoipa_redoc::Redoc::with_url("/redoc", _api.clone()));
 
     #[cfg(feature = "rapidoc")]
     let router = router.merge(
-        utoipa_rapidoc::RapiDoc::with_openapi("/api-docs/rapidoc.json", api.clone())
+        utoipa_rapidoc::RapiDoc::with_openapi("/api-docs/rapidoc.json", _api.clone())
             .path("/rapidoc"),
     );
 
     #[cfg(feature = "scalar")]
-    let router = router.merge(utoipa_scalar::Scalar::with_url("/scalar", api));
+    let router = router.merge(utoipa_scalar::Scalar::with_url("/scalar", _api));
 
-    router
+    middleware::apply(router).merge(metrics_app())
 }
 
 /// Get health of the API.
diff --git a/crates/warden/src/server/middleware.rs b/crates/warden/src/server/middleware.rs
new file mode 100644
index 0000000..2118fcf
--- /dev/null
+++ b/crates/warden/src/server/middleware.rs
@@ -0,0 +1,22 @@
+mod metrics;
+mod trace_layer;
+
+pub use metrics::*;
+pub use trace_layer::*;
+
+use axum::{Router, http::HeaderName, middleware};
+use tower_http::request_id::{MakeRequestUuid, PropagateRequestIdLayer, SetRequestIdLayer};
+
+use crate::server::middleware::apply_metrics_middleware;
+
+pub const REQUEST_ID_HEADER: &str = "x-request-id";
+
+pub fn apply<T: Clone + Send + Sync + 'static>(router: Router<T>) -> Router<T> {
+    let x_request_id = HeaderName::from_static(REQUEST_ID_HEADER);
+
+    let router = router.layer(PropagateRequestIdLayer::new(x_request_id.clone()));
+
+    apply_trace_context_middleware(router)
+        .layer(SetRequestIdLayer::new(x_request_id, MakeRequestUuid))
+        .route_layer(middleware::from_fn(apply_metrics_middleware))
+}
diff --git a/crates/warden/src/server/middleware/metrics.rs b/crates/warden/src/server/middleware/metrics.rs
new file mode 100644
index 0000000..8644160
--- /dev/null
+++ b/crates/warden/src/server/middleware/metrics.rs
@@ -0,0 +1,33 @@
+use std::time::Instant;
+
+use axum::{
+    extract::{MatchedPath, Request},
+    middleware::Next,
+    response::IntoResponse,
+};
+
+pub async fn apply_metrics_middleware(req: Request, next: Next) -> impl IntoResponse {
+    let start = Instant::now();
+    let path = if let Some(matched_path) = req.extensions().get::<MatchedPath>() {
+        matched_path.as_str().to_owned()
+    } else {
+        req.uri().path().to_owned()
+    };
+    let method = req.method().clone();
+
+    let response = next.run(req).await;
+
+    let latency = start.elapsed().as_secs_f64();
+    let status = response.status().as_u16().to_string();
+
+    let labels = [
+        ("method", method.to_string()),
+        ("path", path),
+        ("status", status),
+    ];
+
+    metrics::counter!("http_requests_total", &labels).increment(1);
+    metrics::histogram!("http_requests_duration_seconds", &labels).record(latency);
+
+    response
+}
diff --git a/crates/warden/src/server/middleware/trace_layer.rs b/crates/warden/src/server/middleware/trace_layer.rs
new file mode 100644
index 0000000..5173e8d
--- /dev/null
+++ b/crates/warden/src/server/middleware/trace_layer.rs
@@ -0,0 +1,24 @@
+use axum::{Router, http::Request};
+use tower_http::trace::TraceLayer;
+use tracing::info_span;
+
+use super::REQUEST_ID_HEADER;
+
+pub fn apply_trace_context_middleware<T: Clone + Send + Sync + 'static>(
+    router: Router<T>,
+) -> Router<T> {
+    router.layer(
+        TraceLayer::new_for_http().make_span_with(|request: &Request<_>| {
+            let request_id = request
+                .headers()
+                .get(REQUEST_ID_HEADER)
+                .expect("should have been applied already");
+
+            info_span!(
+                "http_request",
+                request_id = ?request_id,
+                headers = ?request.headers()
+            )
+        }),
+    )
+}
diff --git a/crates/warden/src/server/routes.rs b/crates/warden/src/server/routes.rs
index 771784b..c03a972 100644
--- a/crates/warden/src/server/routes.rs
+++ b/crates/warden/src/server/routes.rs
@@ -1,3 +1,4 @@
+pub mod metrics;
 pub mod processor;
 
 use utoipa::OpenApi;
diff --git a/crates/warden/src/server/routes/metrics.rs b/crates/warden/src/server/routes/metrics.rs
new file mode 100644
index 0000000..9d4af72
--- /dev/null
+++ b/crates/warden/src/server/routes/metrics.rs
@@ -0,0 +1,24 @@
+use std::future::ready;
+
+use axum::{Router, routing::get};
+use metrics_exporter_prometheus::{Matcher, PrometheusBuilder, PrometheusHandle};
+
+pub fn metrics_app() -> Router {
+    let recorder_handle = setup_metrics_recorder();
+    Router::new().route("/metrics", get(move || ready(recorder_handle.render())))
+}
+
+fn setup_metrics_recorder() -> PrometheusHandle {
+    const EXPONENTIAL_SECONDS: &[f64] = &[
+        0.005, 0.01, 0.025, 0.05, 0.1, 0.25, 0.5, 1.0, 2.5, 5.0, 10.0,
+    ];
+
+    PrometheusBuilder::new()
+        .set_buckets_for_metric(
+            Matcher::Full("http_requests_duration_seconds".to_string()),
+            EXPONENTIAL_SECONDS,
+        )
+        .unwrap()
+        .install_recorder()
+        .unwrap()
+}
diff --git a/crates/warden/src/server/routes/processor/pacs008.rs b/crates/warden/src/server/routes/processor/pacs008.rs
index c0b2bb3..e241d38 100644
--- a/crates/warden/src/server/routes/processor/pacs008.rs
+++ b/crates/warden/src/server/routes/processor/pacs008.rs
@@ -61,9 +61,7 @@ pub(super) async fn post_pacs008(
     let end_to_end_id = cdt_trf_tx_inf
         .as_ref()
         .map(|value| value.pmt_id.end_to_end_id.as_str())
-        .ok_or_else(|| {
-            anyhow::anyhow!("missing end_to_end_id id")
-        })?;
+        .ok_or_else(|| anyhow::anyhow!("missing end_to_end_id id"))?;
 
     Ok(String::default())
 }