LINE1 10 20 30 40 50 60 70 80 901use std::{2 collections::HashMap,3 fs::File,4 io::{Cursor, Read},5 net::TcpListener,6 path::Path,7 process::{Command, Stdio},8 time::{SystemTime, UNIX_EPOCH},9};1011use bollard::{12 Docker,13 models::{14 ContainerCreateBody, ContainerSummary, EndpointSettings, HostConfig, HostConfigLogConfig,15 Mount, MountType, NetworkCreateRequest, NetworkingConfig, PortBinding, RestartPolicy,16 RestartPolicyNameEnum, VolumeCreateRequest,17 },18 query_parameters::{19 CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder, RemoveVolumeOptionsBuilder,20 },21};22use flate2::read::GzDecoder;23use futures_util::StreamExt;24use tar::Archive;2526#[test]27fn existing_once_app_can_start_stop_and_exec_without_replacement() {28 let runtime = tokio::runtime::Runtime::new().unwrap();29 runtime.block_on(async {30 let docker = Docker::connect_with_local_defaults().unwrap();31 let namespace = format!("ino-test-{}-{}", std::process::id(), unique_id());32 let container_name = format!("{namespace}-app-probe-123456");33 let background = ino(&[34 "once",35 "--namespace",36 &namespace,37 "background",38 "run",39 "--once",40 "--no-self-update",41 ])42 .expect("run one background check");43 assert!(44 background.status.success(),45 "{}",46 String::from_utf8_lossy(&background.stderr)47 );48 let app_volume = format!("{namespace}-app-probe");49 create_fixture_app(&docker, &container_name, &app_volume).await;5051 let result: Result<(), String> = async {52 let start = ino(&[53 "once",54 "--namespace",55 &namespace,56 "start",57 "probe.localhost",58 ])?;59 if !start.status.success() {60 return Err(String::from_utf8_lossy(&start.stderr).into_owned());61 }62 if is_running(&docker, &container_name).await? != Some(true) {63 return Err("app did not start".into());64 }6566 let exec = Command::new(env!("CARGO_BIN_EXE_ino"))67 .args([68 "once",69 "--namespace",70 &namespace,71 "exec",72 "probe.localhost",73 "sh",74 "-c",75 "printf stdout-message; printf stderr-message >&2; exit 7",76 ])77 .stdin(Stdio::null())78 .output()79 .map_err(|error| error.to_string())?;80 if exec.status.code() != Some(7)81 || !String::from_utf8_lossy(&exec.stdout).contains("stdout-message")82 || !String::from_utf8_lossy(&exec.stderr).contains("stderr-message")83 {84 return Err(format!(85 "exec result was {:?}, stdout={:?}, stderr={:?}",86 exec.status.code(),87 String::from_utf8_lossy(&exec.stdout),88 String::from_utf8_lossy(&exec.stderr)89 ));90 }9192 let stop = ino(&["once", "--namespace", &namespace, "stop", "probe.localhost"])?;93 if !stop.status.success() {94 return Err(String::from_utf8_lossy(&stop.stderr).into_owned());95 }96 if is_running(&docker, &container_name).await? != Some(false) {97 return Err("app did not stop".into());98 }99100 let remove = ino(&[101 "once",102 "--namespace",103 &namespace,104 "remove",105 "probe.localhost",106 ])?;107 if !remove.status.success() {108 return Err(String::from_utf8_lossy(&remove.stderr).into_owned());109 }110 if docker.inspect_volume(&app_volume).await.is_err() {111 return Err("default app removal deleted the persistent volume".into());112 }113 Ok(())114 }115 .await;116117 cleanup_namespace(&docker, &namespace).await;118 assert!(result.is_ok(), "lifecycle integration failed: {result:?}");119 });120}121122#[test]123fn deploys_once_compatible_image_through_isolated_proxy_and_cleans_up() {124 let runtime = tokio::runtime::Runtime::new().unwrap();125 runtime.block_on(async {126 let docker = Docker::connect_with_local_defaults().unwrap();127 let unique = unique_id();128 let namespace = format!("ino-deploy-test-{}-{unique}", std::process::id());129 let proxy_name = format!("{namespace}-proxy");130 let backup_path = std::env::temp_dir().join(format!(131 "ino-backup-test-{}-{unique}.tar.gz",132 std::process::id()133 ));134 let automatic_backup_directory = std::env::temp_dir().join(format!(135 "ino-auto-backup-test-{}-{unique}",136 std::process::id()137 ));138 let ports = (0..3)139 .map(|_| TcpListener::bind(("127.0.0.1", 0)).unwrap())140 .collect::<Vec<_>>();141 let http_port = ports[0].local_addr().unwrap().port();142 let tls_port = ports[1].local_addr().unwrap().port();143 let metrics_port = ports[2].local_addr().unwrap().port();144145 let result: Result<(), String> = async {146 docker147 .create_network(NetworkCreateRequest {148 name: namespace.clone(),149 driver: Some("bridge".to_owned()),150 ..Default::default()151 })152 .await153 .map_err(|error| error.to_string())?;154 docker155 .create_volume(VolumeCreateRequest {156 name: Some(proxy_name.clone()),157 ..Default::default()158 })159 .await160 .map_err(|error| error.to_string())?;161162 let proxy_id = create_proxy_container(163 &docker,164 &namespace,165 &proxy_name,166 http_port,167 tls_port,168 metrics_port,169 )170 .await?;171 drop(ports);172 docker173 .start_container(&proxy_id, None)174 .await175 .map_err(|error| error.to_string())?;176177 let original_app_name =178 deploy_and_verify(&docker, &namespace, &automatic_backup_directory).await?;179 backup_and_remove(&docker, &namespace, &backup_path, &original_app_name).await?;180 restore_and_update(&namespace, &backup_path)?;181 let (updated_name, updated_settings) = verify_updated_app(&docker, &namespace).await?;182 rotate_keys(&docker, &namespace, &updated_settings).await?;183 verify_background_checks(184 &docker,185 &namespace,186 &proxy_name,187 &automatic_backup_directory,188 &updated_name,189 )190 .await?;191 teardown_and_verify(&docker, &namespace).await?;192 Ok(())193 }194 .await;195196 cleanup_namespace(&docker, &namespace).await;197 let _ = std::fs::remove_file(&backup_path);198 let _ = std::fs::remove_dir_all(&automatic_backup_directory);199 assert!(result.is_ok(), "deployment integration failed: {result:?}");200 });201}202203fn unique_id() -> u128 {204 SystemTime::now()205 .duration_since(UNIX_EPOCH)206 .unwrap()207 .as_nanos()208}209210async fn create_fixture_app(docker: &Docker, container_name: &str, app_volume: &str) {211 docker212 .create_volume(VolumeCreateRequest {213 name: Some(app_volume.to_owned()),214 labels: Some(HashMap::from([("once".to_owned(), "{}".to_owned())])),215 ..Default::default()216 })217 .await218 .expect("create app data volume");219 let settings = serde_json::json!({220 "name": "probe",221 "image": "alpine:latest",222 "host": "probe.localhost"223 });224 let labels = HashMap::from([("once".to_owned(), settings.to_string())]);225 let config = ContainerCreateBody {226 image: Some("alpine:latest".to_owned()),227 cmd: Some(vec!["sleep".to_owned(), "600".to_owned()]),228 labels: Some(labels),229 ..Default::default()230 };231 let options = CreateContainerOptionsBuilder::default()232 .name(container_name)233 .build();234 docker235 .create_container(Some(options), config)236 .await237 .expect("create Once-shaped fixture container");238}239240async fn is_running(docker: &Docker, container_name: &str) -> Result<Option<bool>, String> {241 let inspect = docker242 .inspect_container(container_name, None)243 .await244 .map_err(|error| error.to_string())?;245 Ok(inspect.state.and_then(|state| state.running))246}247248async fn create_proxy_container(249 docker: &Docker,250 namespace: &str,251 proxy_name: &str,252 http_port: u16,253 tls_port: u16,254 metrics_port: u16,255) -> Result<String, String> {256 let settings = serde_json::json!({257 "httpPort": http_port,258 "httpsPort": tls_port,259 "metricsPort": metrics_port260 });261 let labels = HashMap::from([("once".to_owned(), settings.to_string())]);262 let port_bindings = HashMap::from([263 (264 "80/tcp".to_owned(),265 Some(vec![PortBinding {266 host_ip: None,267 host_port: Some(http_port.to_string()),268 }]),269 ),270 (271 "443/tcp".to_owned(),272 Some(vec![PortBinding {273 host_ip: None,274 host_port: Some(tls_port.to_string()),275 }]),276 ),277 (278 format!("{metrics_port}/tcp"),279 Some(vec![PortBinding {280 host_ip: Some("127.0.0.1".to_owned()),281 host_port: Some(metrics_port.to_string()),282 }]),283 ),284 ]);285 let proxy_config = ContainerCreateBody {286 image: Some("basecamp/kamal-proxy:once-01".to_owned()),287 cmd: Some(vec![288 "kamal-proxy".to_owned(),289 "run".to_owned(),290 "--metrics-port".to_owned(),291 metrics_port.to_string(),292 ]),293 labels: Some(labels),294 exposed_ports: Some(vec![295 "80/tcp".to_owned(),296 "443/tcp".to_owned(),297 format!("{metrics_port}/tcp"),298 ]),299 host_config: Some(HostConfig {300 port_bindings: Some(port_bindings),301 restart_policy: Some(RestartPolicy {302 name: Some(RestartPolicyNameEnum::ALWAYS),303 maximum_retry_count: None,304 }),305 log_config: Some(HostConfigLogConfig {306 typ: Some("json-file".to_owned()),307 config: Some(HashMap::from([308 ("max-size".to_owned(), "10m".to_owned()),309 ("max-file".to_owned(), "1".to_owned()),310 ])),311 }),312 mounts: Some(vec![Mount {313 typ: Some(MountType::VOLUME),314 source: Some(proxy_name.to_owned()),315 target: Some("/home/kamal-proxy/.config/kamal-proxy".to_owned()),316 ..Default::default()317 }]),318 ..Default::default()319 }),320 networking_config: Some(NetworkingConfig {321 endpoints_config: Some(HashMap::from([(322 namespace.to_owned(),323 EndpointSettings::default(),324 )])),325 }),326 ..Default::default()327 };328 let create_options = CreateContainerOptionsBuilder::default()329 .name(proxy_name)330 .build();331 let proxy = docker332 .create_container(Some(create_options), proxy_config)333 .await334 .map_err(|error| error.to_string())?;335 Ok(proxy.id)336}337338async fn deploy_and_verify(339 docker: &Docker,340 namespace: &str,341 automatic_backup_directory: &Path,342) -> Result<String, String> {343 let deploy = ino(&[344 "once",345 "--namespace",346 namespace,347 "deploy",348 "ghcr.io/basecamp/writebook:latest",349 "--host",350 "localhost",351 "--disable-tls",352 "--env",353 "CUSTOM=port-test",354 "--smtp-server",355 "smtp.example.test",356 "--smtp-port",357 "587",358 "--smtp-username",359 "mailer",360 "--smtp-password",361 "test-password",362 "--smtp-from",363 "test@example.test",364 "--cpus",365 "1",366 "--memory",367 "2048",368 "--auto-update=false",369 "--backup-path",370 automatic_backup_directory.to_str().unwrap(),371 "--auto-backup",372 ])?;373 if !deploy.status.success() {374 return Err(format!(375 "deploy failed: {}",376 String::from_utf8_lossy(&deploy.stderr)377 ));378 }379 let list = ino(&["once", "--namespace", namespace, "list"])?;380 if !list.status.success()381 || !String::from_utf8_lossy(&list.stdout).contains("localhost (running)")382 {383 return Err(format!(384 "deployed app not listed: {}",385 String::from_utf8_lossy(&list.stdout)386 ));387 }388389 let containers = all_containers(docker).await?;390 let settings = settings_for_host(&containers, "localhost")391 .ok_or_else(|| "deployed app settings label not found".to_owned())?;392 if settings["env"]["CUSTOM"] != "port-test"393 || settings["smtp"]["server"] != "smtp.example.test"394 || settings["smtp"]["port"] != "587"395 || settings["resources"]["cpus"] != 1396 || settings["resources"]["memoryMB"] != 2048397 || settings["autoUpdate"] != false398 || settings["backup"]["path"] != automatic_backup_directory.to_string_lossy().as_ref()399 || settings["backup"]["autoBackup"] != true400 {401 return Err(format!(402 "deployment settings were not preserved: {settings}"403 ));404 }405 settings["name"]406 .as_str()407 .map(str::to_owned)408 .ok_or_else(|| "missing app name in settings".to_owned())409}410411async fn backup_and_remove(412 docker: &Docker,413 namespace: &str,414 backup_path: &Path,415 original_app_name: &str,416) -> Result<(), String> {417 let backup = ino(&[418 "once",419 "--namespace",420 namespace,421 "backup",422 "localhost",423 backup_path.to_str().unwrap(),424 ])?;425 if !backup.status.success() {426 return Err(format!(427 "backup failed: {}",428 String::from_utf8_lossy(&backup.stderr)429 ));430 }431 let archive_file = File::open(backup_path).map_err(|error| error.to_string())?;432 let mut archive = Archive::new(GzDecoder::new(archive_file));433 let mut entries = Vec::new();434 let mut archived_app_settings = None;435 for entry in archive.entries().map_err(|error| error.to_string())? {436 let mut entry = entry.map_err(|error| error.to_string())?;437 let path = entry438 .path()439 .map_err(|error| error.to_string())?440 .to_string_lossy()441 .into_owned();442 if path == "once.application.json" {443 let mut contents = String::new();444 entry445 .read_to_string(&mut contents)446 .map_err(|error| error.to_string())?;447 archived_app_settings = serde_json::from_str::<serde_json::Value>(&contents).ok();448 }449 entries.push(path);450 }451 if !entries.iter().any(|path| path == "once.volume.json")452 || !entries.iter().any(|path| path == "data")453 || archived_app_settings454 .as_ref()455 .is_none_or(|value| value["host"] != "localhost")456 {457 return Err(format!(458 "backup archive entries were incomplete: {entries:?}"459 ));460 }461462 let remove = ino(&["once", "--namespace", namespace, "remove", "localhost"])?;463 if !remove.status.success() {464 return Err(format!(465 "remove before restore failed: {}",466 String::from_utf8_lossy(&remove.stderr)467 ));468 }469 let old_volume = format!("{namespace}-app-{original_app_name}");470 if docker.inspect_volume(&old_volume).await.is_err() {471 return Err("removing app without --remove-data lost its volume".into());472 }473 docker474 .remove_volume(475 &old_volume,476 Some(RemoveVolumeOptionsBuilder::default().force(true).build()),477 )478 .await479 .map_err(|error| error.to_string())480}481482fn restore_and_update(namespace: &str, backup_path: &Path) -> Result<(), String> {483 let restore = ino(&[484 "once",485 "--namespace",486 namespace,487 "restore",488 backup_path.to_str().unwrap(),489 ])?;490 if !restore.status.success() {491 return Err(format!(492 "restore failed: {}",493 String::from_utf8_lossy(&restore.stderr)494 ));495 }496 let list = ino(&["once", "--namespace", namespace, "list"])?;497 if !list.status.success()498 || !String::from_utf8_lossy(&list.stdout).contains("localhost (running)")499 {500 return Err(format!(501 "restored app not listed: {}",502 String::from_utf8_lossy(&list.stdout)503 ));504 }505506 let update = ino(&[507 "once",508 "--namespace",509 namespace,510 "update",511 "localhost",512 "--smtp-from",513 "updated@example.test",514 "--auto-update=true",515 ])?;516 if !update.status.success() {517 return Err(format!(518 "update failed: {}",519 String::from_utf8_lossy(&update.stderr)520 ));521 }522 let updated_list = ino(&["once", "--namespace", namespace, "list"])?;523 if !updated_list.status.success()524 || !String::from_utf8_lossy(&updated_list.stdout).contains("localhost (running)")525 {526 return Err("updated app was not discoverable".into());527 }528 Ok(())529}530531async fn verify_updated_app(532 docker: &Docker,533 namespace: &str,534) -> Result<(String, serde_json::Value), String> {535 let containers = all_containers(docker).await?;536 let updated_settings = settings_for_host(&containers, "localhost")537 .ok_or_else(|| "updated settings label not found".to_owned())?;538 if updated_settings["smtp"]["from"] != "updated@example.test"539 || updated_settings["autoUpdate"] != true540 {541 return Err(format!(542 "update settings were not applied: {updated_settings}"543 ));544 }545 let updated_name = updated_settings["name"]546 .as_str()547 .ok_or_else(|| "updated app name missing".to_owned())?548 .to_owned();549 let app_prefix = format!("{namespace}-app-{updated_name}-");550 let app_containers = containers551 .iter()552 .filter(|container| {553 container.names.as_ref().is_some_and(|names| {554 names555 .iter()556 .any(|name| name.trim_start_matches('/').starts_with(&app_prefix))557 })558 })559 .count();560 if app_containers != 1 {561 return Err(format!("update left {app_containers} app containers"));562 }563 let updated_volume = format!("{namespace}-app-{updated_name}");564 if docker.inspect_volume(&updated_volume).await.is_err() {565 return Err("update did not preserve the app data volume".into());566 }567 Ok((updated_name, updated_settings))568}569570async fn rotate_keys(571 docker: &Docker,572 namespace: &str,573 updated_settings: &serde_json::Value,574) -> Result<(), String> {575 let old_vapid_public = updated_settings["keys"]["vapidPublicKey"]576 .as_str()577 .unwrap_or_default()578 .to_owned();579 let reset = ino(&[580 "once",581 "--namespace",582 namespace,583 "keys",584 "reset",585 "localhost",586 "--vapid",587 ])?;588 if !reset.status.success() {589 return Err(format!(590 "key reset failed: {}",591 String::from_utf8_lossy(&reset.stderr)592 ));593 }594 let reset_settings = app_settings_for_host(docker, "localhost").await?;595 if reset_settings["keys"]["vapidPublicKey"] == old_vapid_public {596 return Err("VAPID reset did not change the public key".into());597 }598 let vapid_private = reset_settings["keys"]["vapidPrivateKey"]599 .as_str()600 .ok_or_else(|| "reset VAPID private key missing".to_owned())?;601 let set_vapid_arg = format!("--vapid={vapid_private}");602 let set = ino(&[603 "once",604 "--namespace",605 namespace,606 "keys",607 "set",608 "localhost",609 "--secret-key-base",610 "replacement-secret-key-base",611 &set_vapid_arg,612 ])?;613 if !set.status.success() {614 return Err(format!(615 "key set failed: {}",616 String::from_utf8_lossy(&set.stderr)617 ));618 }619 let set_settings = app_settings_for_host(docker, "localhost").await?;620 if set_settings["keys"]["secretKeyBase"] != "replacement-secret-key-base"621 || set_settings["keys"]["vapidPrivateKey"] != vapid_private622 {623 return Err("key set did not persist the requested key values".into());624 }625 Ok(())626}627628async fn verify_background_checks(629 docker: &Docker,630 namespace: &str,631 proxy_name: &str,632 automatic_backup_directory: &Path,633 updated_name: &str,634) -> Result<(), String> {635 std::fs::create_dir_all(automatic_backup_directory).map_err(|error| error.to_string())?;636 let background = ino(&[637 "once",638 "--namespace",639 namespace,640 "background",641 "run",642 "--once",643 "--no-self-update",644 ])?;645 if !background.status.success() {646 return Err(format!(647 "background check failed: {}",648 String::from_utf8_lossy(&background.stderr)649 ));650 }651 if std::fs::read_dir(automatic_backup_directory)652 .map_err(|error| error.to_string())?653 .next()654 .is_none()655 {656 return Err("background check did not create an automatic backup".into());657 }658659 let state_options = bollard::query_parameters::DownloadFromContainerOptionsBuilder::default()660 .path("/home/kamal-proxy/.config/kamal-proxy/once-state.json")661 .build();662 let mut state_stream = docker.download_from_container(proxy_name, Some(state_options));663 let mut state_tar = Vec::new();664 while let Some(chunk) = state_stream.next().await {665 state_tar.extend_from_slice(&chunk.map_err(|error| error.to_string())?);666 }667 let mut state_archive = tar::Archive::new(Cursor::new(state_tar));668 let mut state_json = String::new();669 for entry in state_archive.entries().map_err(|error| error.to_string())? {670 let mut entry = entry.map_err(|error| error.to_string())?;671 if entry672 .path()673 .map_err(|error| error.to_string())?674 .to_string_lossy()675 == "once-state.json"676 {677 entry678 .read_to_string(&mut state_json)679 .map_err(|error| error.to_string())?;680 }681 }682 let state: serde_json::Value =683 serde_json::from_str(&state_json).map_err(|error| error.to_string())?;684 if state["apps"][updated_name]["lastBackup"]["at"]685 .as_str()686 .unwrap_or_default()687 .is_empty()688 || state["apps"][updated_name]["lastUpdate"]["at"]689 .as_str()690 .unwrap_or_default()691 .is_empty()692 {693 return Err(format!(694 "operation history missing from Once state: {state}"695 ));696 }697 Ok(())698}699700async fn teardown_and_verify(docker: &Docker, namespace: &str) -> Result<(), String> {701 let teardown = ino(&[702 "once",703 "--namespace",704 namespace,705 "teardown",706 "--remove-data",707 ])?;708 if !teardown.status.success() {709 return Err(format!(710 "teardown failed: {}",711 String::from_utf8_lossy(&teardown.stderr)712 ));713 }714 let containers = all_containers(docker).await?;715 if containers.iter().any(|container| {716 container.names.as_ref().is_some_and(|names| {717 names.iter().any(|name| {718 name.trim_start_matches('/')719 .starts_with(&format!("{namespace}-"))720 })721 })722 }) {723 return Err("teardown left namespace containers behind".into());724 }725 let volumes = docker726 .list_volumes(Some(727 bollard::query_parameters::ListVolumesOptionsBuilder::default().build(),728 ))729 .await730 .map_err(|error| error.to_string())?;731 if volumes732 .volumes733 .unwrap_or_default()734 .iter()735 .any(|volume| volume.name.starts_with(&format!("{namespace}-")))736 {737 return Err("teardown with --remove-data left namespace volumes behind".into());738 }739 Ok(())740}741742async fn cleanup_namespace(docker: &Docker, namespace: &str) {743 let containers = docker744 .list_containers(Some(745 bollard::query_parameters::ListContainersOptionsBuilder::default()746 .all(true)747 .build(),748 ))749 .await750 .unwrap_or_default();751 for container in containers {752 let Some(names) = container.names else {753 continue;754 };755 if names.iter().any(|name| {756 name.trim_start_matches('/')757 .starts_with(&format!("{namespace}-"))758 }) && let Some(id) = container.id759 {760 let _ = docker761 .remove_container(762 &id,763 Some(RemoveContainerOptionsBuilder::default().force(true).build()),764 )765 .await;766 }767 }768769 if let Ok(volumes) = docker770 .list_volumes(Some(771 bollard::query_parameters::ListVolumesOptionsBuilder::default().build(),772 ))773 .await774 {775 for volume in volumes.volumes.unwrap_or_default() {776 if volume.name.starts_with(&format!("{namespace}-")) {777 let _ = docker778 .remove_volume(779 &volume.name,780 Some(RemoveVolumeOptionsBuilder::default().force(true).build()),781 )782 .await;783 }784 }785 }786 let _ = docker.remove_network(namespace).await;787}788789async fn all_containers(docker: &Docker) -> Result<Vec<ContainerSummary>, String> {790 docker791 .list_containers(Some(792 bollard::query_parameters::ListContainersOptionsBuilder::default()793 .all(true)794 .build(),795 ))796 .await797 .map_err(|error| error.to_string())798}799800fn settings_for_host(containers: &[ContainerSummary], host: &str) -> Option<serde_json::Value> {801 containers802 .iter()803 .filter_map(|container| container.labels.as_ref())804 .find_map(|labels| {805 labels806 .get("once")807 .and_then(|label| serde_json::from_str::<serde_json::Value>(label).ok())808 .filter(|value| value["host"] == host)809 })810}811812async fn app_settings_for_host(docker: &Docker, host: &str) -> Result<serde_json::Value, String> {813 let containers = all_containers(docker).await?;814 settings_for_host(&containers, host)815 .ok_or_else(|| format!("no app metadata found for host {host}"))816}817818fn ino(args: &[&str]) -> Result<std::process::Output, String> {819 Command::new(env!("CARGO_BIN_EXE_ino"))820 .args(args)821 .stdin(Stdio::null())822 .output()823 .map_err(|error| error.to_string())824}