split service&instance metrics, serve metrics for build-server & watcher

src/bin/cratesfyi.rs

@@ -1,5 +1,6 @@
 use std::env;
 use std::fmt::Write;
+use std::net::SocketAddr;
 use std::path::PathBuf;
 use std::str::FromStr;
 use std::sync::Arc;
@@ -14,8 +15,8 @@ use docs_rs::utils::{
     remove_crate_priority, set_crate_priority, ConfigName,
 };
 use docs_rs::{
-    start_web_server, BuildQueue, Config, Context, Index, Metrics, PackageKind, RustwideBuilder,
-    Storage,
+    start_background_metrics_webserver, start_web_server, BuildQueue, Config, Context, Index,
+    InstanceMetrics, PackageKind, RustwideBuilder, ServiceMetrics, Storage,
 };
 use humantime::Duration;
 use once_cell::sync::OnceCell;
@@ -107,10 +108,12 @@ enum CommandLine {
     /// Starts web server
     StartWebServer {
         #[arg(name = "SOCKET_ADDR", default_value = "0.0.0.0:3000")]
-        socket_addr: String,
+        socket_addr: SocketAddr,
     },
 
     StartRegistryWatcher {
+        #[arg(name = "SOCKET_ADDR", default_value = "0.0.0.0:3000")]
+        metric_server_socket_addr: SocketAddr,
         /// Enable or disable the repository stats updater
         #[arg(
             long = "repository-stats-updater",
@@ -122,7 +125,10 @@ enum CommandLine {
         cdn_invalidator: Toggle,
     },
 
-    StartBuildServer,
+    StartBuildServer {
+        #[arg(name = "SOCKET_ADDR", default_value = "0.0.0.0:3000")]
+        metric_server_socket_addr: SocketAddr,
+    },
 
     /// Starts the daemon
     Daemon {
@@ -154,6 +160,7 @@ impl CommandLine {
                 subcommand,
             } => subcommand.handle_args(ctx, skip_if_exists)?,
             Self::StartRegistryWatcher {
+                metric_server_socket_addr,
                 repository_stats_updater,
                 cdn_invalidator,
             } => {
@@ -164,16 +171,22 @@ impl CommandLine {
                     docs_rs::utils::daemon::start_background_cdn_invalidator(&ctx)?;
                 }
 
+                start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?;
+
                 docs_rs::utils::watch_registry(ctx.build_queue()?, ctx.config()?, ctx.index()?)?;
             }
-            Self::StartBuildServer => {
+            Self::StartBuildServer {
+                metric_server_socket_addr,
+            } => {
+                start_background_metrics_webserver(Some(metric_server_socket_addr), &ctx)?;
+
                 let build_queue = ctx.build_queue()?;
                 let rustwide_builder = RustwideBuilder::init(&ctx)?;
                 queue_builder(rustwide_builder, build_queue)?;
             }
             Self::StartWebServer { socket_addr } => {
                 // Blocks indefinitely
-                start_web_server(Some(&socket_addr), &ctx)?;
+                start_web_server(Some(socket_addr), &ctx)?;
             }
             Self::Daemon { registry_watcher } => {
                 docs_rs::utils::start_daemon(ctx, registry_watcher == Toggle::Enabled)?;
@@ -699,7 +712,8 @@ struct BinContext {
     cdn: OnceCell<Arc<CdnBackend>>,
     config: OnceCell<Arc<Config>>,
     pool: OnceCell<Pool>,
-    metrics: OnceCell<Arc<Metrics>>,
+    service_metrics: OnceCell<Arc<ServiceMetrics>>,
+    instance_metrics: OnceCell<Arc<InstanceMetrics>>,
     index: OnceCell<Arc<Index>>,
     repository_stats_updater: OnceCell<Arc<RepositoryStatsUpdater>>,
     runtime: OnceCell<Arc<Runtime>>,
@@ -713,7 +727,8 @@ impl BinContext {
             cdn: OnceCell::new(),
             config: OnceCell::new(),
             pool: OnceCell::new(),
-            metrics: OnceCell::new(),
+            service_metrics: OnceCell::new(),
+            instance_metrics: OnceCell::new(),
             index: OnceCell::new(),
             repository_stats_updater: OnceCell::new(),
             runtime: OnceCell::new(),
@@ -740,13 +755,13 @@ impl Context for BinContext {
     lazy! {
         fn build_queue(self) -> BuildQueue = BuildQueue::new(
             self.pool()?,
-            self.metrics()?,
+            self.instance_metrics()?,
             self.config()?,
             self.storage()?,
         );
         fn storage(self) -> Storage = Storage::new(
             self.pool()?,
-            self.metrics()?,
+            self.instance_metrics()?,
             self.config()?,
             self.runtime()?,
         )?;
@@ -755,7 +770,8 @@ impl Context for BinContext {
             &self.runtime()?,
         );
         fn config(self) -> Config = Config::from_env()?;
-        fn metrics(self) -> Metrics = Metrics::new()?;
+        fn service_metrics(self) -> ServiceMetrics = ServiceMetrics::new()?;
+        fn instance_metrics(self) -> InstanceMetrics = InstanceMetrics::new()?;
         fn runtime(self) -> Runtime = {
             Builder::new_multi_thread()
                 .enable_all()
@@ -780,7 +796,9 @@ impl Context for BinContext {
     fn pool(&self) -> Result<Pool> {
         Ok(self
             .pool
-            .get_or_try_init::<_, Error>(|| Ok(Pool::new(&*self.config()?, self.metrics()?)?))?
+            .get_or_try_init::<_, Error>(|| {
+                Ok(Pool::new(&*self.config()?, self.instance_metrics()?)?)
+            })?
             .clone())
     }
 }

src/build_queue.rs

@@ -4,7 +4,7 @@ use crate::docbuilder::PackageKind;
 use crate::error::Result;
 use crate::storage::Storage;
 use crate::utils::{get_config, get_crate_priority, report_error, retry, set_config, ConfigName};
-use crate::{Config, Index, Metrics, RustwideBuilder};
+use crate::{Config, Index, InstanceMetrics, RustwideBuilder};
 use anyhow::Context;
 use fn_error_context::context;
 
@@ -28,14 +28,14 @@ pub struct BuildQueue {
     config: Arc<Config>,
     storage: Arc<Storage>,
     pub(crate) db: Pool,
-    metrics: Arc<Metrics>,
+    metrics: Arc<InstanceMetrics>,
     max_attempts: i32,
 }
 
 impl BuildQueue {
     pub fn new(
         db: Pool,
-        metrics: Arc<Metrics>,
+        metrics: Arc<InstanceMetrics>,
         config: Arc<Config>,
         storage: Arc<Storage>,
     ) -> Self {
@@ -600,7 +600,7 @@ mod tests {
             assert!(!called, "there were still items in the queue");
 
             // Ensure metrics were recorded correctly
-            let metrics = env.metrics();
+            let metrics = env.instance_metrics();
             assert_eq!(metrics.total_builds.get(), 9);
             assert_eq!(metrics.failed_builds.get(), 1);
 

src/cdn.rs

@@ -1,4 +1,4 @@
-use crate::{metrics::duration_to_seconds, utils::report_error, Config, Metrics};
+use crate::{metrics::duration_to_seconds, utils::report_error, Config, InstanceMetrics};
 use anyhow::{anyhow, bail, Context, Error, Result};
 use aws_sdk_cloudfront::{
     config::{retry::RetryConfig, Region},
@@ -303,7 +303,7 @@ impl CdnBackend {
 #[instrument(skip(conn))]
 pub(crate) fn handle_queued_invalidation_requests(
     cdn: &CdnBackend,
-    metrics: &Metrics,
+    metrics: &InstanceMetrics,
     conn: &mut impl postgres::GenericClient,
     distribution_id: &str,
 ) -> Result<()> {
@@ -707,13 +707,13 @@ mod tests {
             // now handle the queued invalidations
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_static",
             )?;
@@ -741,13 +741,13 @@ mod tests {
             // now handle again
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_static",
             )?;
@@ -806,7 +806,7 @@ mod tests {
             // handle the queued invalidations
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
@@ -860,7 +860,7 @@ mod tests {
             // handle the queued invalidations
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
@@ -886,7 +886,7 @@ mod tests {
             // now handle again
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;
@@ -921,7 +921,7 @@ mod tests {
             // run the handler
             handle_queued_invalidation_requests(
                 &env.cdn(),
-                &env.metrics(),
+                &env.instance_metrics(),
                 &mut *conn,
                 "distribution_id_web",
             )?;

src/context.rs

@@ -2,7 +2,7 @@ use crate::cdn::CdnBackend;
 use crate::db::Pool;
 use crate::error::Result;
 use crate::repositories::RepositoryStatsUpdater;
-use crate::{BuildQueue, Config, Index, Metrics, Storage};
+use crate::{BuildQueue, Config, Index, InstanceMetrics, ServiceMetrics, Storage};
 use std::sync::Arc;
 use tokio::runtime::Runtime;
 
@@ -12,7 +12,8 @@ pub trait Context {
     fn storage(&self) -> Result<Arc<Storage>>;
     fn cdn(&self) -> Result<Arc<CdnBackend>>;
     fn pool(&self) -> Result<Pool>;
-    fn metrics(&self) -> Result<Arc<Metrics>>;
+    fn service_metrics(&self) -> Result<Arc<ServiceMetrics>>;
+    fn instance_metrics(&self) -> Result<Arc<InstanceMetrics>>;
     fn index(&self) -> Result<Arc<Index>>;
     fn repository_stats_updater(&self) -> Result<Arc<RepositoryStatsUpdater>>;
     fn runtime(&self) -> Result<Arc<Runtime>>;

src/db/pool.rs

@@ -1,4 +1,4 @@
-use crate::metrics::Metrics;
+use crate::metrics::InstanceMetrics;
 use crate::Config;
 use postgres::{Client, NoTls};
 use r2d2_postgres::PostgresConnectionManager;
@@ -15,12 +15,12 @@ pub struct Pool {
     pool: Arc<std::sync::Mutex<Option<r2d2::Pool<PostgresConnectionManager<NoTls>>>>>,
     #[cfg(not(test))]
     pool: r2d2::Pool<PostgresConnectionManager<NoTls>>,
-    metrics: Arc<Metrics>,
+    metrics: Arc<InstanceMetrics>,
     max_size: u32,
 }
 
 impl Pool {
-    pub fn new(config: &Config, metrics: Arc<Metrics>) -> Result<Pool, PoolError> {
+    pub fn new(config: &Config, metrics: Arc<InstanceMetrics>) -> Result<Pool, PoolError> {
         debug!(
             "creating database pool (if this hangs, consider running `docker-compose up -d db s3`)"
         );
@@ -30,13 +30,17 @@ impl Pool {
     #[cfg(test)]
     pub(crate) fn new_with_schema(
         config: &Config,
-        metrics: Arc<Metrics>,
+        metrics: Arc<InstanceMetrics>,
         schema: &str,
     ) -> Result<Pool, PoolError> {
         Self::new_inner(config, metrics, schema)
     }
 
-    fn new_inner(config: &Config, metrics: Arc<Metrics>, schema: &str) -> Result<Pool, PoolError> {
+    fn new_inner(
+        config: &Config,
+        metrics: Arc<InstanceMetrics>,
+        schema: &str,
+    ) -> Result<Pool, PoolError> {
         let url = config
             .database_url
             .parse()

src/docbuilder/rustwide_builder.rs

@@ -13,7 +13,7 @@ use crate::utils::{
 };
 use crate::RUSTDOC_STATIC_STORAGE_PREFIX;
 use crate::{db::blacklist::is_blacklisted, utils::MetadataPackage};
-use crate::{Config, Context, Index, Metrics, Storage};
+use crate::{Config, Context, Index, InstanceMetrics, Storage};
 use anyhow::{anyhow, bail, Context as _, Error};
 use docsrs_metadata::{BuildTargets, Metadata, DEFAULT_TARGETS, HOST_TARGET};
 use failure::Error as FailureError;
@@ -45,7 +45,7 @@ pub struct RustwideBuilder {
     config: Arc<Config>,
     db: Pool,
     storage: Arc<Storage>,
-    metrics: Arc<Metrics>,
+    metrics: Arc<InstanceMetrics>,
     index: Arc<Index>,
     rustc_version: String,
     repository_stats_updater: Arc<RepositoryStatsUpdater>,
@@ -92,7 +92,7 @@ impl RustwideBuilder {
             config,
             db: context.pool()?,
             storage: context.storage()?,
-            metrics: context.metrics()?,
+            metrics: context.instance_metrics()?,
             index: context.index()?,
             rustc_version: String::new(),
             repository_stats_updater: context.repository_stats_updater()?,

src/lib.rs

@@ -8,9 +8,9 @@ pub use self::context::Context;
 pub use self::docbuilder::PackageKind;
 pub use self::docbuilder::RustwideBuilder;
 pub use self::index::Index;
-pub use self::metrics::Metrics;
+pub use self::metrics::{InstanceMetrics, ServiceMetrics};
 pub use self::storage::Storage;
-pub use self::web::start_web_server;
+pub use self::web::{start_background_metrics_webserver, start_web_server};
 
 mod build_queue;
 pub mod cdn;

src/metrics/macros.rs

@@ -2,7 +2,6 @@ pub(super) trait MetricFromOpts: Sized {
     fn from_opts(opts: prometheus::Opts) -> Result<Self, prometheus::Error>;
 }
 
-#[macro_export]
 macro_rules! metrics {
     (
         $vis:vis struct $name:ident {
@@ -82,7 +81,6 @@ macro_rules! metrics {
     };
 }
 
-#[macro_export]
 macro_rules! load_metric_type {
     ($name:ident as single) => {
         use prometheus::$name;