feat(rocksdb): Add last-ditch destructive database recovery option

src/config/mod.rs

@@ -32,6 +32,8 @@ pub struct Config {
     pub conduit_cache_capacity_modifier: f64,
     #[serde(default = "default_rocksdb_max_open_files")]
     pub rocksdb_max_open_files: i32,
+    #[serde(default = "false_fn")]
+    pub rocksdb_destructive_recovery: bool,
     #[serde(default = "default_pdu_cache_capacity")]
     pub pdu_cache_capacity: u32,
     #[serde(default = "default_cleanup_second_interval")]

src/database/abstraction/rocksdb.rs

@@ -56,7 +56,14 @@ impl KeyValueDatabaseEngine for Arc<Engine> {
         let cache_capacity_bytes = (config.db_cache_capacity_mb * 1024.0 * 1024.0) as usize;
         let rocksdb_cache = rocksdb::Cache::new_lru_cache(cache_capacity_bytes).unwrap();
 
-        let db_opts = db_options(config.rocksdb_max_open_files, &rocksdb_cache);
+        let mut db_opts = db_options(config.rocksdb_max_open_files, &rocksdb_cache);
+
+        // Destructive database recovery
+        // Last-ditch effort to bring database to usable state, generally should not be needed
+        // Ignores/Discards corrupted WAL entries
+        if config.rocksdb_destructive_recovery {
+            db_opts.set_wal_recovery_mode(rocksdb::DBRecoveryMode::SkipAnyCorruptedRecord);
+        }
 
         let cfs = rocksdb::DBWithThreadMode::<rocksdb::MultiThreaded>::list_cf(
             &db_opts,

src/service/globals/mod.rs

@@ -254,6 +254,10 @@ impl Service {
         self.config.enable_lightning_bolt
     }
 
+    pub fn rocksdb_destructive_recovery(&self) -> bool {
+        self.config.rocksdb_destructive_recovery
+    }
+
     pub fn trusted_servers(&self) -> &[OwnedServerName] {
         &self.config.trusted_servers
     }