Run daemon without registry index watcher in local docker

.env.sample

@@ -1,5 +1,3 @@
-CRATESFYI_GITHUB_USERNAME=
-CRATESFYI_GITHUB_ACCESSTOKEN=
 CRATESFYI_PREFIX=ignored/cratesfyi-prefix
 CRATESFYI_DATABASE_URL=postgresql://cratesfyi:password@localhost
 RUST_LOG=cratesfyi,rustwide=info

Cargo.toml

@@ -44,6 +44,8 @@ zstd = "0.5"
 git2 = { version = "0.13.6", default-features = false }
 path-slash = "0.1.3"
 once_cell = { version = "1.4.0", features = ["parking_lot"] }
+base64 = "0.12.1"
+strum = { version = "0.18.0", features = ["derive"] }
 
 # Data serialization and deserialization
 serde = { version = "1.0", features = ["derive"] }

dockerfiles/Dockerfile

@@ -79,4 +79,4 @@ COPY dockerfiles/entrypoint.sh /opt/docsrs/
 
 WORKDIR /opt/docsrs
 ENTRYPOINT ["/opt/docsrs/entrypoint.sh"]
-CMD ["start-web-server"]
+CMD ["daemon", "--registry-watcher=disabled"]

src/bin/cratesfyi.rs

@@ -8,6 +8,7 @@ use cratesfyi::{BuildQueue, Config, DocBuilder, DocBuilderOptions, RustwideBuild
 use failure::Error;
 use once_cell::sync::OnceCell;
 use structopt::StructOpt;
+use strum::VariantNames;
 
 pub fn main() -> Result<(), Error> {
     let _ = dotenv::dotenv();
@@ -40,6 +41,13 @@ fn logger_init() {
     rustwide::logging::init_with(builder.build());
 }
 
+#[derive(Debug, Clone, Copy, PartialEq, Eq, strum::EnumString, strum::EnumVariantNames)]
+#[strum(serialize_all = "snake_case")]
+enum Toggle {
+    Enabled,
+    Disabled,
+}
+
 #[derive(Debug, Clone, PartialEq, Eq, StructOpt)]
 #[structopt(
     name = "cratesfyi",
@@ -65,6 +73,14 @@ enum CommandLine {
         /// Deprecated. Run the server in the foreground instead of detaching a child
         #[structopt(name = "FOREGROUND", short = "f", long = "foreground")]
         foreground: bool,
+
+        /// Enable or disable the registry watcher to automatically enqueue newly published crates
+        #[structopt(
+            long = "registry-watcher",
+            default_value = "enabled",
+            possible_values(Toggle::VARIANTS)
+        )]
+        registry_watcher: Toggle,
     },
 
     /// Database operations
@@ -98,12 +114,20 @@ impl CommandLine {
                     ctx.build_queue()?,
                 )?;
             }
-            Self::Daemon { foreground } => {
+            Self::Daemon {
+                foreground,
+                registry_watcher,
+            } => {
                 if foreground {
                     log::warn!("--foreground was passed, but there is no need for it anymore");
                 }
 
-                cratesfyi::utils::start_daemon(ctx.config()?, ctx.pool()?, ctx.build_queue()?)?;
+                cratesfyi::utils::start_daemon(
+                    ctx.config()?,
+                    ctx.pool()?,
+                    ctx.build_queue()?,
+                    registry_watcher == Toggle::Enabled,
+                )?;
             }
             Self::Database { subcommand } => subcommand.handle_args(ctx)?,
             Self::Queue { subcommand } => subcommand.handle_args(ctx)?,
@@ -430,8 +454,8 @@ impl DatabaseSubcommand {
             }
 
             Self::UpdateGithubFields => {
-                cratesfyi::utils::github_updater(&*ctx.conn()?)
-                    .expect("Failed to update github fields");
+                cratesfyi::utils::GithubUpdater::new(&*ctx.config()?, ctx.pool()?)?
+                    .update_all_crates()?;
             }
 
             Self::AddDirectory { directory, prefix } => {

src/config.rs

@@ -12,6 +12,10 @@ pub struct Config {
     pub(crate) max_pool_size: u32,
     pub(crate) min_pool_idle: u32,
 
+    // Github authentication
+    pub(crate) github_username: Option<String>,
+    pub(crate) github_accesstoken: Option<String>,
+
     // Max size of the files served by the docs.rs frontend
     pub(crate) max_file_size: usize,
     pub(crate) max_file_size_html: usize,
@@ -26,10 +30,20 @@ impl Config {
             max_pool_size: env("DOCSRS_MAX_POOL_SIZE", 90)?,
             min_pool_idle: env("DOCSRS_MIN_POOL_IDLE", 10)?,
 
+            github_username: maybe_env("CRATESFYI_GITHUB_USERNAME")?,
+            github_accesstoken: maybe_env("CRATESFYI_GITHUB_ACCESSTOKEN")?,
+
             max_file_size: env("DOCSRS_MAX_FILE_SIZE", 50 * 1024 * 1024)?,
             max_file_size_html: env("DOCSRS_MAX_FILE_SIZE_HTML", 5 * 1024 * 1024)?,
         })
     }
+
+    pub fn github_auth(&self) -> Option<(&str, &str)> {
+        Some((
+            self.github_username.as_deref()?,
+            self.github_accesstoken.as_deref()?,
+        ))
+    }
 }
 
 fn env<T>(var: &str, default: T) -> Result<T, Error>
@@ -58,7 +72,10 @@ where
             .parse::<T>()
             .map(Some)
             .with_context(|_| format!("failed to parse configuration variable {}", var))?),
-        Err(VarError::NotPresent) => Ok(None),
+        Err(VarError::NotPresent) => {
+            log::debug!("optional configuration variable {} is not set", var);
+            Ok(None)
+        }
         Err(VarError::NotUnicode(_)) => bail!("configuration variable {} is not UTF-8", var),
     }
 }