Skip to content

Commit 0e8b32b

Browse files
committed
ci: [#254] add container healthcheck for index service
For both services: - The Index API - The Tracker Stattistics Importer (console cronjob) An API was added for the Importer. The Importer API has two endpoints: - GET /health_check -> to get the crobjob status. It only checks that is running periodically. - POST /heartbeat -> used by the cronjob to inform it's alive. And a new endpoint was added the the Index API: - GET /health_check -> to check API responsiveness.
1 parent 04b70ba commit 0e8b32b

File tree

16 files changed

+220
-34
lines changed

16 files changed

+220
-34
lines changed

Containerfile

+7-2
Original file line numberDiff line numberDiff line change
@@ -85,7 +85,9 @@ COPY --from=build \
8585
RUN cargo nextest run --workspace-remap /test/src/ --extract-to /test/src/ --no-run --archive-file /test/torrust-index.tar.zst
8686
RUN cargo nextest run --workspace-remap /test/src/ --target-dir-remap /test/src/target/ --cargo-metadata /test/src/target/nextest/cargo-metadata.json --binaries-metadata /test/src/target/nextest/binaries-metadata.json
8787

88-
RUN mkdir -p /app/bin/; cp -l /test/src/target/release/torrust-index /app/bin/torrust-index
88+
RUN mkdir -p /app/bin/; \
89+
cp -l /test/src/target/release/torrust-index /app/bin/torrust-index; \
90+
cp -l /test/src/target/release/health_check /app/bin/health_check;
8991
# RUN mkdir -p /app/lib/; cp -l $(realpath $(ldd /app/bin/torrust-index | grep "libz\.so\.1" | awk '{print $3}')) /app/lib/libz.so.1
9092
RUN chown -R root:root /app; chmod -R u=rw,go=r,a+X /app; chmod -R a+x /app/bin
9193

@@ -99,11 +101,13 @@ ARG TORRUST_INDEX_PATH_CONFIG="/etc/torrust/index/index.toml"
99101
ARG TORRUST_INDEX_DATABASE_DRIVER="sqlite3"
100102
ARG USER_ID=1000
101103
ARG API_PORT=3001
104+
ARG IMPORTER_API_PORT=3002
102105

103106
ENV TORRUST_INDEX_PATH_CONFIG=${TORRUST_INDEX_PATH_CONFIG}
104107
ENV TORRUST_INDEX_DATABASE_DRIVER=${TORRUST_INDEX_DATABASE_DRIVER}
105108
ENV USER_ID=${USER_ID}
106109
ENV API_PORT=${API_PORT}
110+
ENV IMPORTER_API_PORT=${IMPORTER_API_PORT}
107111
ENV TZ=Etc/UTC
108112

109113
EXPOSE ${API_PORT}/tcp
@@ -130,5 +134,6 @@ CMD ["sh"]
130134
FROM runtime as release
131135
ENV RUNTIME="release"
132136
COPY --from=test /app/ /usr/
133-
# HEALTHCHECK CMD ["/usr/bin/wget", "--no-verbose", "--tries=1", "--spider", "localhost:${API_PORT}/version"]
137+
HEALTHCHECK --interval=5s --timeout=5s --start-period=3s --retries=3 \
138+
CMD /usr/bin/health_check http://localhost:${API_PORT}/health_check && /usr/bin/health_check http://localhost:${IMPORTER_API_PORT}/health_check || exit 1
134139
CMD ["/usr/bin/torrust-index"]

contrib/dev-tools/container/e2e/mysql/run-e2e-tests.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -25,10 +25,10 @@ echo "Running E2E tests using MySQL ..."
2525
./contrib/dev-tools/container/e2e/mysql/e2e-env-up.sh || exit 1
2626

2727
# Wait for conatiners to be healthy
28-
./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3
29-
# todo: implement healthchecks for tracker and index and wait until they are healthy
30-
#wait_for_container torrust-tracker-1 10 3
31-
#wait_for_container torrust-idx-back-1 10 3
28+
./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3 || exit 1
29+
# todo: implement healthchecks for the tracker and wait until it's healthy
30+
#./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-tracker-1 10 3
31+
./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-index-1 10 3 || exit 1
3232
sleep 20s
3333

3434
# Just to make sure that everything is up and running

contrib/dev-tools/container/e2e/sqlite/run-e2e-tests.sh

+4-4
Original file line numberDiff line numberDiff line change
@@ -26,10 +26,10 @@ echo "Running E2E tests using SQLite ..."
2626
./contrib/dev-tools/container/e2e/sqlite/e2e-env-up.sh || exit 1
2727

2828
# Wait for conatiners to be healthy
29-
./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3
30-
# todo: implement healthchecks for tracker and index and wait until they are healthy
31-
#wait_for_container torrust-tracker-1 10 3
32-
#wait_for_container torrust-idx-back-1 10 3
29+
./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-mysql-1 10 3 || exit 1
30+
# todo: implement healthchecks for the tracker and wait until it's healthy
31+
#./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-tracker-1 10 3
32+
./contrib/dev-tools/container/functions/wait_for_container_to_be_healthy.sh torrust-index-1 10 3 || exit 1
3333
sleep 20s
3434

3535
# Just to make sure that everything is up and running

share/default/config/index.container.mysql.toml

+1
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,4 @@ max_torrent_page_size = 30
4848

4949
[tracker_statistics_importer]
5050
torrent_info_update_interval = 3600
51+
port = 3002

share/default/config/index.container.sqlite3.toml

+1
Original file line numberDiff line numberDiff line change
@@ -48,3 +48,4 @@ max_torrent_page_size = 30
4848

4949
[tracker_statistics_importer]
5050
torrent_info_update_interval = 3600
51+
port = 3002

share/default/config/index.development.sqlite3.toml

+1
Original file line numberDiff line numberDiff line change
@@ -44,3 +44,4 @@ max_torrent_page_size = 30
4444

4545
[tracker_statistics_importer]
4646
torrent_info_update_interval = 3600
47+
port = 3002

src/app.rs

+15-22
Original file line numberDiff line numberDiff line change
@@ -19,8 +19,8 @@ use crate::services::user::{self, DbBannedUserList, DbUserProfileRepository, DbU
1919
use crate::services::{proxy, settings, torrent};
2020
use crate::tracker::statistics_importer::StatisticsImporter;
2121
use crate::web::api::v1::auth::Authentication;
22-
use crate::web::api::{start, Version};
23-
use crate::{mailer, tracker};
22+
use crate::web::api::Version;
23+
use crate::{console, mailer, tracker, web};
2424

2525
pub struct Running {
2626
pub api_socket_addr: SocketAddr,
@@ -46,8 +46,12 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running
4646

4747
let settings = configuration.settings.read().await;
4848

49+
// From [database] config
4950
let database_connect_url = settings.database.connect_url.clone();
50-
let torrent_info_update_interval = settings.tracker_statistics_importer.torrent_info_update_interval;
51+
// From [importer] config
52+
let importer_torrent_info_update_interval = settings.tracker_statistics_importer.torrent_info_update_interval;
53+
let importer_port = settings.tracker_statistics_importer.port;
54+
// From [net] config
5155
let net_ip = "0.0.0.0".to_string();
5256
let net_port = settings.net.port;
5357

@@ -153,29 +157,18 @@ pub async fn run(configuration: Configuration, api_version: &Version) -> Running
153157
ban_service,
154158
));
155159

156-
// Start repeating task to import tracker torrent data and updating
160+
// Start cronjob to import tracker torrent data and updating
157161
// seeders and leechers info.
158-
159-
let weak_tracker_statistics_importer = Arc::downgrade(&tracker_statistics_importer);
160-
161-
let tracker_statistics_importer_handle = tokio::spawn(async move {
162-
let interval = std::time::Duration::from_secs(torrent_info_update_interval);
163-
let mut interval = tokio::time::interval(interval);
164-
interval.tick().await; // first tick is immediate...
165-
loop {
166-
interval.tick().await;
167-
if let Some(tracker) = weak_tracker_statistics_importer.upgrade() {
168-
drop(tracker.import_all_torrents_statistics().await);
169-
} else {
170-
break;
171-
}
172-
}
173-
});
162+
let tracker_statistics_importer_handle = console::tracker_statistics_importer::start(
163+
importer_port,
164+
importer_torrent_info_update_interval,
165+
&tracker_statistics_importer,
166+
);
174167

175168
// Start API server
169+
let running_api = web::api::start(app_data, &net_ip, net_port, api_version).await;
176170

177-
let running_api = start(app_data, &net_ip, net_port, api_version).await;
178-
171+
// Full running application
179172
Running {
180173
api_socket_addr: running_api.socket_addr,
181174
api_server: running_api.api_server,

src/bin/health_check.rs

+37
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,37 @@
1+
//! Minimal `curl` or `wget` to be used for container health checks.
2+
//!
3+
//! It's convenient to avoid using third-party libraries because:
4+
//!
5+
//! - They are harder to maintain.
6+
//! - They introduce new attack vectors.
7+
use std::{env, process};
8+
9+
#[tokio::main]
10+
async fn main() {
11+
let args: Vec<String> = env::args().collect();
12+
if args.len() != 2 {
13+
eprintln!("Usage: cargo run --bin health_check <HEALTH_URL>");
14+
eprintln!("Example: cargo run --bin health_check http://localhost:3002/health_check");
15+
std::process::exit(1);
16+
}
17+
18+
println!("Health check ...");
19+
20+
let url = &args[1].clone();
21+
22+
match reqwest::get(url).await {
23+
Ok(response) => {
24+
if response.status().is_success() {
25+
println!("STATUS: {}", response.status());
26+
process::exit(0);
27+
} else {
28+
println!("Non-success status received.");
29+
process::exit(1);
30+
}
31+
}
32+
Err(err) => {
33+
println!("ERROR: {err}");
34+
process::exit(1);
35+
}
36+
}
37+
}

src/config.rs

+3
Original file line numberDiff line numberDiff line change
@@ -334,12 +334,15 @@ impl Default for Api {
334334
pub struct TrackerStatisticsImporter {
335335
/// The interval in seconds to get statistics from the tracker.
336336
pub torrent_info_update_interval: u64,
337+
/// The port the Importer API is listening on. Default to `3002`.
338+
pub port: u16,
337339
}
338340

339341
impl Default for TrackerStatisticsImporter {
340342
fn default() -> Self {
341343
Self {
342344
torrent_info_update_interval: 3600,
345+
port: 3002,
343346
}
344347
}
345348
}

src/console/mod.rs

+1
Original file line numberDiff line numberDiff line change
@@ -1 +1,2 @@
11
pub mod commands;
2+
pub(crate) mod tracker_statistics_importer;
+130
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,130 @@
1+
//! Cronjob to import tracker torrent data and updating seeders and leechers
2+
//! info.
3+
//!
4+
//! It has two services:
5+
//!
6+
//! - The importer which is the cronjob executed at regular intervals.
7+
//! - The importer API.
8+
//!
9+
//! The cronjob sends a heartbeat signal to the API each time it is executed.
10+
//! The last heartbeat signal time is used to determine whether the cronjob was
11+
//! executed successfully or not. The API has a `health_check` endpoint which is
12+
//! used when the application is running in containers.
13+
use std::sync::{Arc, Mutex};
14+
15+
use axum::extract::State;
16+
use axum::routing::{get, post};
17+
use axum::{Json, Router};
18+
use chrono::{DateTime, Utc};
19+
use log::{error, info};
20+
use serde_json::{json, Value};
21+
use tokio::task::JoinHandle;
22+
23+
use crate::tracker::statistics_importer::StatisticsImporter;
24+
25+
const IMPORTER_API_IP: &str = "127.0.0.1";
26+
27+
#[derive(Clone)]
28+
struct ImporterState {
29+
/// Shared variable to store the timestamp of the last heartbeat sent
30+
/// by the cronjob.
31+
pub last_heartbeat: Arc<Mutex<DateTime<Utc>>>,
32+
/// Interval between importation executions
33+
pub torrent_info_update_interval: u64,
34+
}
35+
36+
pub fn start(
37+
importer_port: u16,
38+
torrent_info_update_interval: u64,
39+
tracker_statistics_importer: &Arc<StatisticsImporter>,
40+
) -> JoinHandle<()> {
41+
let weak_tracker_statistics_importer = Arc::downgrade(tracker_statistics_importer);
42+
43+
tokio::spawn(async move {
44+
info!("Tracker statistics importer launcher started");
45+
46+
// Start the Importer API
47+
48+
let _importer_api_handle = tokio::spawn(async move {
49+
let import_state = Arc::new(ImporterState {
50+
last_heartbeat: Arc::new(Mutex::new(Utc::now())),
51+
torrent_info_update_interval,
52+
});
53+
54+
let app = Router::new()
55+
.route("/", get(|| async { Json(json!({})) }))
56+
.route("/health_check", get(health_check_handler))
57+
.with_state(import_state.clone())
58+
.route("/heartbeat", post(heartbeat_handler))
59+
.with_state(import_state);
60+
61+
let addr = format!("{IMPORTER_API_IP}:{importer_port}");
62+
63+
info!("Tracker statistics importer API server listening on http://{}", addr);
64+
65+
axum::Server::bind(&addr.parse().unwrap())
66+
.serve(app.into_make_service())
67+
.await
68+
.unwrap();
69+
});
70+
71+
// Start the Importer cronjob
72+
73+
info!("Tracker statistics importer cronjob starting ...");
74+
75+
let interval = std::time::Duration::from_secs(torrent_info_update_interval);
76+
let mut interval = tokio::time::interval(interval);
77+
78+
interval.tick().await; // first tick is immediate...
79+
80+
loop {
81+
interval.tick().await;
82+
83+
info!("Running tracker statistics importer ...");
84+
85+
if let Err(e) = send_heartbeat(importer_port).await {
86+
error!("Failed to send heartbeat from importer cronjob: {}", e);
87+
}
88+
89+
if let Some(tracker) = weak_tracker_statistics_importer.upgrade() {
90+
drop(tracker.import_all_torrents_statistics().await);
91+
} else {
92+
break;
93+
}
94+
}
95+
})
96+
}
97+
98+
/// Endpoint for container health check.
99+
async fn health_check_handler(State(state): State<Arc<ImporterState>>) -> Json<Value> {
100+
let margin_in_seconds = 10;
101+
let now = Utc::now();
102+
let last_heartbeat = state.last_heartbeat.lock().unwrap();
103+
104+
if now.signed_duration_since(*last_heartbeat).num_seconds()
105+
<= (state.torrent_info_update_interval + margin_in_seconds).try_into().unwrap()
106+
{
107+
Json(json!({ "status": "Ok" }))
108+
} else {
109+
Json(json!({ "status": "Error" }))
110+
}
111+
}
112+
113+
/// The tracker statistics importer cronjob sends a heartbeat on each execution
114+
/// to inform that it's alive. This endpoint handles receiving that signal.
115+
async fn heartbeat_handler(State(state): State<Arc<ImporterState>>) -> Json<Value> {
116+
let now = Utc::now();
117+
let mut last_heartbeat = state.last_heartbeat.lock().unwrap();
118+
*last_heartbeat = now;
119+
Json(json!({ "status": "Heartbeat received" }))
120+
}
121+
122+
/// Send a heartbeat from the importer cronjob to the importer API.
123+
async fn send_heartbeat(importer_port: u16) -> Result<(), reqwest::Error> {
124+
let client = reqwest::Client::new();
125+
let url = format!("http://{IMPORTER_API_IP}:{importer_port}/heartbeat");
126+
127+
client.post(url).send().await?;
128+
129+
Ok(())
130+
}

src/lib.rs

+1
Original file line numberDiff line numberDiff line change
@@ -200,6 +200,7 @@
200200
//!
201201
//! [tracker_statistics_importer]
202202
//! torrent_info_update_interval = 3600
203+
//! port = 3002
203204
//! ```
204205
//!
205206
//! For more information about configuration you can visit the documentation for the [`config`]) module.

src/web/api/v1/contexts/settings/mod.rs

+1
Original file line numberDiff line numberDiff line change
@@ -75,6 +75,7 @@
7575
//! },
7676
//! "tracker_statistics_importer": {
7777
//! "torrent_info_update_interval": 3600
78+
//! "port": 3002
7879
//! }
7980
//! }
8081
//! }

src/web/api/v1/routes.rs

+9-2
Original file line numberDiff line numberDiff line change
@@ -4,7 +4,8 @@ use std::sync::Arc;
44

55
use axum::extract::DefaultBodyLimit;
66
use axum::routing::get;
7-
use axum::Router;
7+
use axum::{Json, Router};
8+
use serde_json::{json, Value};
89
use tower_http::compression::CompressionLayer;
910
use tower_http::cors::CorsLayer;
1011

@@ -34,7 +35,8 @@ pub fn router(app_data: Arc<AppData>) -> Router {
3435
.nest("/proxy", proxy::routes::router(app_data.clone()));
3536

3637
let router = Router::new()
37-
.route("/", get(about_page_handler).with_state(app_data))
38+
.route("/", get(about_page_handler).with_state(app_data.clone()))
39+
.route("/health_check", get(health_check_handler).with_state(app_data))
3840
.nest(&format!("/{API_VERSION_URL_PREFIX}"), v1_api_routes);
3941

4042
let router = if env::var(ENV_VAR_CORS_PERMISSIVE).is_ok() {
@@ -45,3 +47,8 @@ pub fn router(app_data: Arc<AppData>) -> Router {
4547

4648
router.layer(DefaultBodyLimit::max(10_485_760)).layer(CompressionLayer::new())
4749
}
50+
51+
/// Endpoint for container health check.
52+
async fn health_check_handler() -> Json<Value> {
53+
Json(json!({ "status": "Ok" }))
54+
}

tests/common/contexts/settings/mod.rs

+2
Original file line numberDiff line numberDiff line change
@@ -82,6 +82,7 @@ pub struct Api {
8282
#[derive(Deserialize, Serialize, PartialEq, Debug, Clone)]
8383
pub struct TrackerStatisticsImporter {
8484
pub torrent_info_update_interval: u64,
85+
port: u16,
8586
}
8687

8788
impl From<DomainSettings> for Settings {
@@ -185,6 +186,7 @@ impl From<DomainTrackerStatisticsImporter> for TrackerStatisticsImporter {
185186
fn from(tracker_statistics_importer: DomainTrackerStatisticsImporter) -> Self {
186187
Self {
187188
torrent_info_update_interval: tracker_statistics_importer.torrent_info_update_interval,
189+
port: tracker_statistics_importer.port,
188190
}
189191
}
190192
}

0 commit comments

Comments
 (0)