spider_util/constants/
middleware.rs

1//! Default values used by built-in middleware.
2
3/// Initial delay for adaptive rate limiting (500ms).
4pub const RATE_LIMIT_INITIAL_DELAY_MS: u64 = 500;
5
6/// Minimum delay for rate limiting (50ms).
7pub const RATE_LIMIT_MIN_DELAY_MS: u64 = 50;
8
9/// Maximum delay for rate limiting (60 seconds).
10pub const RATE_LIMIT_MAX_DELAY_MS: u64 = 60_000;
11
12/// Maximum jitter for rate limiting (500ms).
13pub const RATE_LIMIT_MAX_JITTER_MS: u64 = 500;
14
15/// Error penalty multiplier for adaptive rate limiting.
16pub const RATE_LIMIT_ERROR_PENALTY_MULTIPLIER: f64 = 1.5;
17
18/// Success decay multiplier for adaptive rate limiting.
19pub const RATE_LIMIT_SUCCESS_DECAY_MULTIPLIER: f64 = 0.95;
20
21/// Forbidden penalty multiplier for adaptive rate limiting.
22pub const RATE_LIMIT_FORBIDDEN_PENALTY_MULTIPLIER: f64 = 1.2;
23
24/// Default cache TTL for middleware (1 hour).
25pub const MIDDLEWARE_CACHE_TTL_SECS: u64 = 3600;
26
27/// Default cache capacity for middleware.
28pub const MIDDLEWARE_CACHE_CAPACITY: u64 = 10_000;
29
30/// Default retry attempts for retry middleware.
31pub const RETRY_DEFAULT_MAX_RETRIES: u32 = 3;
32
33/// Default backoff factor for retry middleware.
34pub const RETRY_DEFAULT_BACKOFF_FACTOR: f64 = 1.0;
35
36/// Default maximum delay for retry middleware (3 minutes).
37pub const RETRY_DEFAULT_MAX_DELAY_MS: u64 = 180_000;
38
39/// Default HTTP status codes to retry.
40pub const RETRY_DEFAULT_HTTP_CODES: &[u16] = &[500, 502, 503, 504, 408, 429];
spider_util/constants/middleware.rs

spider_util/constants/
middleware.rs
