diff --git a/Cargo.lock b/Cargo.lock index 96b4a3b..3a83e22 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -7322,7 +7322,7 @@ checksum = "a2eb9349b6444b326872e140eb1cf5e7c522154d69e7a0ffb0fb81c06b37543f" [[package]] name = "storb_base" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "async-trait", @@ -7357,7 +7357,7 @@ dependencies = [ [[package]] name = "storb_cli" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "chrono", @@ -7385,7 +7385,7 @@ dependencies = [ [[package]] name = "storb_miner" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "axum", @@ -7413,7 +7413,7 @@ dependencies = [ [[package]] name = "storb_validator" -version = "0.2.2" +version = "0.2.3" dependencies = [ "anyhow", "axum", diff --git a/Cargo.toml b/Cargo.toml index 010196c..47fb42c 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -4,7 +4,7 @@ members = ["crates/*"] default-members = ["crates/storb_cli"] [workspace.package] -version = "0.2.2" +version = "0.2.3" description = "An object storage subnet on the Bittensor network" edition = "2021" authors = ["Storb Technologies Ltd"] diff --git a/crates/storb_base/src/constants.rs b/crates/storb_base/src/constants.rs index fcfcbc0..33b2b38 100644 --- a/crates/storb_base/src/constants.rs +++ b/crates/storb_base/src/constants.rs @@ -16,6 +16,6 @@ pub const CHUNK_M: usize = 8; pub const INFO_REQ_TIMEOUT: Duration = Duration::from_secs(5); /// Timeout params for upload requests -pub const MIN_BANDWIDTH: u64 = 20 * 1024; // minimum "bandwidth" +pub const MIN_BANDWIDTH: u64 = 100 * 1024; // 100 KiB/s pub const SYNC_BUFFER_SIZE: usize = 32; diff --git a/crates/storb_miner/Cargo.toml b/crates/storb_miner/Cargo.toml index 2a675a3..599274f 100644 --- a/crates/storb_miner/Cargo.toml +++ b/crates/storb_miner/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "storb_miner" -version = "0.2.2" +version = "0.2.3" edition.workspace = true [dependencies] diff --git a/crates/storb_validator/Cargo.toml b/crates/storb_validator/Cargo.toml index 43503cf..7c85e81 100644 --- a/crates/storb_validator/Cargo.toml +++ b/crates/storb_validator/Cargo.toml @@ -1,6 +1,6 @@ [package] name = "storb_validator" -version = "0.2.2" +version = "0.2.3" edition.workspace = true [dependencies] diff --git a/crates/storb_validator/src/constants.rs b/crates/storb_validator/src/constants.rs index ea167d4..b886b9b 100644 --- a/crates/storb_validator/src/constants.rs +++ b/crates/storb_validator/src/constants.rs @@ -19,9 +19,9 @@ pub const SYNTHETIC_CHALLENGE_FREQUENCY: u64 = 300; pub const MAX_CHALLENGE_PIECE_NUM: i32 = 5; pub const SYNTH_CHALLENGE_TIMEOUT: f64 = 1.0; // TODO: modify this pub const SYNTH_CHALLENGE_WAIT_BEFORE_RETRIEVE: f64 = 3.0; -pub const MIN_SYNTH_CHUNK_SIZE: usize = 1024 * 10 * 10; // minimum size of synthetic data in bytes -pub const MAX_SYNTH_CHUNK_SIZE: usize = 1024 * 10 * 10 * 10 * 10; // maximum size of synthetic data in bytes -pub const MAX_SYNTH_CHALLENGE_MINER_NUM: usize = 25; // maximum number of miners to challenge +pub const MIN_SYNTH_CHUNK_SIZE: usize = 512 * 1024; // minimum size of synthetic data in bytes +pub const MAX_SYNTH_CHUNK_SIZE: usize = 8 * 1024 * 1024; // maximum size of synthetic data in bytes +pub const MAX_SYNTH_CHALLENGE_MINER_NUM: usize = 50; // maximum number of miners to challenge // constants for MetadataDB pub const DB_MAX_LIFETIME: u64 = 3600; // Close connections after 1 hour @@ -34,5 +34,7 @@ pub const INFO_API_RATE_LIMIT_MAX_REQUESTS: usize = 10; // Initial values for alpha and beta used in the scoring system // These were empirically derived to minimise reliable node churn -pub const INITIAL_ALPHA: f64 = 500.0; -pub const INITIAL_BETA: f64 = 1000.0; +pub const INITIAL_ALPHA: f64 = 18.0; +pub const INITIAL_BETA: f64 = 36.0; +pub const LAMBDA: f64 = 0.99; +pub const AUDIT_WEIGHT: f64 = 1.0; diff --git a/crates/storb_validator/src/download.rs b/crates/storb_validator/src/download.rs index c3176c3..839b48a 100644 --- a/crates/storb_validator/src/download.rs +++ b/crates/storb_validator/src/download.rs @@ -17,6 +17,7 @@ use tokio::sync::{mpsc, Mutex, RwLock}; use tokio_stream::StreamExt; use tracing::{debug, error, trace}; +use crate::constants::AUDIT_WEIGHT; use crate::metadata; use crate::metadata::db::MetadataDBCommand; use crate::scoring::ScoringSystem; @@ -34,7 +35,7 @@ pub async fn update_scoring_on_failure( ) -> Result<()> { let mut scoring_system_rw = scoring_system.write().await; scoring_system_rw - .update_alpha_beta_db(miner_uid, 1.0, false) + .update_alpha_beta_db(miner_uid, AUDIT_WEIGHT, false) .await .map_err(|e| anyhow!("Failed to update scoring system: {}", e))?; drop(scoring_system_rw); @@ -238,7 +239,7 @@ impl DownloadProcessor { Err(e) => { let mut scoring_system_rw = scoring_system.write().await; scoring_system_rw - .update_alpha_beta_db(miner_uid, 1.0, false) + .update_alpha_beta_db(miner_uid, AUDIT_WEIGHT, false) .await .map_err(|e| anyhow!("Failed to update scoring system: {}", e))?; drop(scoring_system_rw); @@ -272,7 +273,7 @@ impl DownloadProcessor { Err(e) => { let mut scoring_system_rw = scoring_system.write().await; scoring_system_rw - .update_alpha_beta_db(miner_uid, 1.0, false) + .update_alpha_beta_db(miner_uid, AUDIT_WEIGHT, false) .await .map_err(|e| anyhow!("Failed to update scoring system: {}", e))?; drop(scoring_system_rw); @@ -285,7 +286,7 @@ impl DownloadProcessor { db.conn.lock().await.execute("UPDATE miner_stats SET total_successes = total_successes + 1 WHERE miner_uid = $1", [&miner_uid])?; let mut scoring_system_rw = scoring_system.write().await; scoring_system_rw - .update_alpha_beta_db(miner_uid, 1.0, true) + .update_alpha_beta_db(miner_uid, AUDIT_WEIGHT, true) .await .map_err(|e| anyhow!("Failed to update scoring system: {}", e))?; drop(scoring_system_rw); diff --git a/crates/storb_validator/src/scoring.rs b/crates/storb_validator/src/scoring.rs index fbf44f5..e087aa7 100644 --- a/crates/storb_validator/src/scoring.rs +++ b/crates/storb_validator/src/scoring.rs @@ -9,7 +9,7 @@ use rusqlite::params; use serde::{Deserialize, Serialize}; use tracing::{debug, error, info, warn}; -use crate::constants::{INITIAL_ALPHA, INITIAL_BETA}; +use crate::constants::{INITIAL_ALPHA, INITIAL_BETA, LAMBDA}; /// ScoreState stores the scores for each miner. /// @@ -39,6 +39,7 @@ pub struct ScoringSystem { pub lambda: f64, } +// normalization function that produces an s curve #[inline] pub fn normalize_min_max(arr: &Array1) -> Array1 { let min = arr.iter().cloned().fold(f64::INFINITY, f64::min); @@ -100,7 +101,7 @@ impl ScoringSystem { state_file: scoring_state_file.to_path_buf(), initial_alpha: INITIAL_ALPHA, initial_beta: INITIAL_BETA, - lambda: 0.99, // forgetting factor for challenges + lambda: LAMBDA, // forgetting factor for challenges }; match scoring_system.load_state() { @@ -163,7 +164,7 @@ impl ScoringSystem { )?; // calculate new alpha and beta - let (new_beta, new_alpha) = get_new_alpha_beta(beta, alpha, 0.99, weight, success); + let (new_beta, new_alpha) = get_new_alpha_beta(beta, alpha, LAMBDA, weight, success); // update alpha and beta in the database conn.execute( "UPDATE miner_stats SET alpha = ?1, beta = ?2 WHERE miner_uid = ?3", diff --git a/crates/storb_validator/src/upload.rs b/crates/storb_validator/src/upload.rs index d779f38..4d3eaa4 100644 --- a/crates/storb_validator/src/upload.rs +++ b/crates/storb_validator/src/upload.rs @@ -25,7 +25,7 @@ use subxt::ext::codec::Compact; use tokio::sync::{mpsc, RwLock}; use tracing::{debug, error, info, trace, warn}; -use crate::constants::MIN_REQUIRED_MINERS; +use crate::constants::{AUDIT_WEIGHT, MIN_REQUIRED_MINERS}; use crate::metadata; use crate::metadata::models::SqlAccountId; use crate::quic::establish_miner_connections; @@ -79,7 +79,6 @@ pub async fn upload_piece_data( message, }; let payload_bytes = bincode::serialize(&payload)?; - debug!("payload_bytes: {:?}", payload_bytes); let piece_size = data.len(); let size: f64 = piece_size as f64; @@ -101,11 +100,6 @@ pub async fn upload_piece_data( send_stream.write_all(&data).await?; send_stream.finish()?; - debug!( - "Timeout duration for upload acknowledgement: {} milliseconds", - timeout_duration.as_millis() - ); - // Wrap the upload response operation in a timeout let hash = tokio::time::timeout(timeout_duration, async { // Read hash response @@ -316,16 +310,6 @@ impl<'a> UploadProcessor<'a> { ..infohash_value }; - // display chunks_with_pieces - debug!("Chunks with pieces: {:?}", chunks_with_pieces); - - // display hash of chunks_with_pieces - let serialized = bincode::serialize(&chunks_with_pieces)?; - debug!( - "HASH: {}", - hex::encode(blake3::hash(&serialized).as_bytes()) - ); - match metadata::db::MetadataDB::insert_object( &metadatadb_sender, signed_infohash_value, @@ -691,12 +675,12 @@ pub async fn upload_piece_to_miner( )?; let mut scoring_system_guard = scoring_system.write().await; scoring_system_guard - .update_alpha_beta_db(miner_uid, 1.0, true) + .update_alpha_beta_db(miner_uid, AUDIT_WEIGHT, true) .await?; } else { let mut scoring_system_guard = scoring_system.write().await; scoring_system_guard - .update_alpha_beta_db(miner_uid, 1.0, false) + .update_alpha_beta_db(miner_uid, AUDIT_WEIGHT, false) .await?; } diff --git a/migrations/scoresdb/20250222003351_db.up.sql b/migrations/scoresdb/20250222003351_db.up.sql index ea74d50..23ea133 100644 --- a/migrations/scoresdb/20250222003351_db.up.sql +++ b/migrations/scoresdb/20250222003351_db.up.sql @@ -1,8 +1,8 @@ -- Table for miner stats -- CREATE TABLE miner_stats ( miner_uid INTEGER PRIMARY KEY, - alpha FLOAT DEFAULT 500.0, - beta FLOAT DEFAULT 1000.0, + alpha FLOAT DEFAULT 18.0, + beta FLOAT DEFAULT 36.0, challenge_successes INTEGER DEFAULT 0, challenge_attempts INTEGER DEFAULT 0, retrieval_successes INTEGER DEFAULT 0, diff --git a/settings.toml.example b/settings.toml.example index 0e64327..ba31fd2 100644 --- a/settings.toml.example +++ b/settings.toml.example @@ -1,4 +1,4 @@ -version = "0.2.2" +version = "0.2.3" log_level = "INFO" # Must be one of TRACE, DEBUG, INFO, WARN, ERROR netuid = 26 # Testnet is 269