Skip to content
1        10        20        30        40        50        60        70        80        901use std::{2    collections::HashMap,3    fs::File,4    io::{Cursor, Read},5    net::TcpListener,6    path::Path,7    process::{Command, Stdio},8    time::{SystemTime, UNIX_EPOCH},9};1011use bollard::{12    Docker,13    models::{14        ContainerCreateBody, ContainerSummary, EndpointSettings, HostConfig, HostConfigLogConfig,15        Mount, MountType, NetworkCreateRequest, NetworkingConfig, PortBinding, RestartPolicy,16        RestartPolicyNameEnum, VolumeCreateRequest,17    },18    query_parameters::{19        CreateContainerOptionsBuilder, RemoveContainerOptionsBuilder, RemoveVolumeOptionsBuilder,20    },21};22use flate2::read::GzDecoder;23use futures_util::StreamExt;24use tar::Archive;2526#[test]27fn existing_once_app_can_start_stop_and_exec_without_replacement() {28    let runtime = tokio::runtime::Runtime::new().unwrap();29    runtime.block_on(async {30        let docker = Docker::connect_with_local_defaults().unwrap();31        let namespace = format!("ino-test-{}-{}", std::process::id(), unique_id());32        let container_name = format!("{namespace}-app-probe-123456");33        let background = ino(&[34            "once",35            "--namespace",36            &namespace,37            "background",38            "run",39            "--once",40            "--no-self-update",41        ])42        .expect("run one background check");43        assert!(44            background.status.success(),45            "{}",46            String::from_utf8_lossy(&background.stderr)47        );48        let app_volume = format!("{namespace}-app-probe");49        create_fixture_app(&docker, &container_name, &app_volume).await;5051        let result: Result<(), String> = async {52            let start = ino(&[53                "once",54                "--namespace",55                &namespace,56                "start",57                "probe.localhost",58            ])?;59            if !start.status.success() {60                return Err(String::from_utf8_lossy(&start.stderr).into_owned());61            }62            if is_running(&docker, &container_name).await? != Some(true) {63                return Err("app did not start".into());64            }6566            let exec = Command::new(env!("CARGO_BIN_EXE_ino"))67                .args([68                    "once",69                    "--namespace",70                    &namespace,71                    "exec",72                    "probe.localhost",73                    "sh",74                    "-c",75                    "printf stdout-message; printf stderr-message >&2; exit 7",76                ])77                .stdin(Stdio::null())78                .output()79                .map_err(|error| error.to_string())?;80            if exec.status.code() != Some(7)81                || !String::from_utf8_lossy(&exec.stdout).contains("stdout-message")82                || !String::from_utf8_lossy(&exec.stderr).contains("stderr-message")83            {84                return Err(format!(85                    "exec result was {:?}, stdout={:?}, stderr={:?}",86                    exec.status.code(),87                    String::from_utf8_lossy(&exec.stdout),88                    String::from_utf8_lossy(&exec.stderr)89                ));90            }9192            let stop = ino(&["once", "--namespace", &namespace, "stop", "probe.localhost"])?;93            if !stop.status.success() {94                return Err(String::from_utf8_lossy(&stop.stderr).into_owned());95            }96            if is_running(&docker, &container_name).await? != Some(false) {97                return Err("app did not stop".into());98            }99100            let remove = ino(&[101                "once",102                "--namespace",103                &namespace,104                "remove",105                "probe.localhost",106            ])?;107            if !remove.status.success() {108                return Err(String::from_utf8_lossy(&remove.stderr).into_owned());109            }110            if docker.inspect_volume(&app_volume).await.is_err() {111                return Err("default app removal deleted the persistent volume".into());112            }113            Ok(())114        }115        .await;116117        cleanup_namespace(&docker, &namespace).await;118        assert!(result.is_ok(), "lifecycle integration failed: {result:?}");119    });120}121122#[test]123fn deploys_once_compatible_image_through_isolated_proxy_and_cleans_up() {124    let runtime = tokio::runtime::Runtime::new().unwrap();125    runtime.block_on(async {126        let docker = Docker::connect_with_local_defaults().unwrap();127        let unique = unique_id();128        let namespace = format!("ino-deploy-test-{}-{unique}", std::process::id());129        let proxy_name = format!("{namespace}-proxy");130        let backup_path = std::env::temp_dir().join(format!(131            "ino-backup-test-{}-{unique}.tar.gz",132            std::process::id()133        ));134        let automatic_backup_directory = std::env::temp_dir().join(format!(135            "ino-auto-backup-test-{}-{unique}",136            std::process::id()137        ));138        let ports = (0..3)139            .map(|_| TcpListener::bind(("127.0.0.1", 0)).unwrap())140            .collect::<Vec<_>>();141        let http_port = ports[0].local_addr().unwrap().port();142        let tls_port = ports[1].local_addr().unwrap().port();143        let metrics_port = ports[2].local_addr().unwrap().port();144145        let result: Result<(), String> = async {146            docker147                .create_network(NetworkCreateRequest {148                    name: namespace.clone(),149                    driver: Some("bridge".to_owned()),150                    ..Default::default()151                })152                .await153                .map_err(|error| error.to_string())?;154            docker155                .create_volume(VolumeCreateRequest {156                    name: Some(proxy_name.clone()),157                    ..Default::default()158                })159                .await160                .map_err(|error| error.to_string())?;161162            let proxy_id = create_proxy_container(163                &docker,164                &namespace,165                &proxy_name,166                http_port,167                tls_port,168                metrics_port,169            )170            .await?;171            drop(ports);172            docker173                .start_container(&proxy_id, None)174                .await175                .map_err(|error| error.to_string())?;176177            let original_app_name =178                deploy_and_verify(&docker, &namespace, &automatic_backup_directory).await?;179            backup_and_remove(&docker, &namespace, &backup_path, &original_app_name).await?;180            restore_and_update(&namespace, &backup_path)?;181            let (updated_name, updated_settings) = verify_updated_app(&docker, &namespace).await?;182            rotate_keys(&docker, &namespace, &updated_settings).await?;183            verify_background_checks(184                &docker,185                &namespace,186                &proxy_name,187                &automatic_backup_directory,188                &updated_name,189            )190            .await?;191            teardown_and_verify(&docker, &namespace).await?;192            Ok(())193        }194        .await;195196        cleanup_namespace(&docker, &namespace).await;197        let _ = std::fs::remove_file(&backup_path);198        let _ = std::fs::remove_dir_all(&automatic_backup_directory);199        assert!(result.is_ok(), "deployment integration failed: {result:?}");200    });201}202203fn unique_id() -> u128 {204    SystemTime::now()205        .duration_since(UNIX_EPOCH)206        .unwrap()207        .as_nanos()208}209210async fn create_fixture_app(docker: &Docker, container_name: &str, app_volume: &str) {211    docker212        .create_volume(VolumeCreateRequest {213            name: Some(app_volume.to_owned()),214            labels: Some(HashMap::from([("once".to_owned(), "{}".to_owned())])),215            ..Default::default()216        })217        .await218        .expect("create app data volume");219    let settings = serde_json::json!({220        "name": "probe",221        "image": "alpine:latest",222        "host": "probe.localhost"223    });224    let labels = HashMap::from([("once".to_owned(), settings.to_string())]);225    let config = ContainerCreateBody {226        image: Some("alpine:latest".to_owned()),227        cmd: Some(vec!["sleep".to_owned(), "600".to_owned()]),228        labels: Some(labels),229        ..Default::default()230    };231    let options = CreateContainerOptionsBuilder::default()232        .name(container_name)233        .build();234    docker235        .create_container(Some(options), config)236        .await237        .expect("create Once-shaped fixture container");238}239240async fn is_running(docker: &Docker, container_name: &str) -> Result<Option<bool>, String> {241    let inspect = docker242        .inspect_container(container_name, None)243        .await244        .map_err(|error| error.to_string())?;245    Ok(inspect.state.and_then(|state| state.running))246}247248async fn create_proxy_container(249    docker: &Docker,250    namespace: &str,251    proxy_name: &str,252    http_port: u16,253    tls_port: u16,254    metrics_port: u16,255) -> Result<String, String> {256    let settings = serde_json::json!({257        "httpPort": http_port,258        "httpsPort": tls_port,259        "metricsPort": metrics_port260    });261    let labels = HashMap::from([("once".to_owned(), settings.to_string())]);262    let port_bindings = HashMap::from([263        (264            "80/tcp".to_owned(),265            Some(vec![PortBinding {266                host_ip: None,267                host_port: Some(http_port.to_string()),268            }]),269        ),270        (271            "443/tcp".to_owned(),272            Some(vec![PortBinding {273                host_ip: None,274                host_port: Some(tls_port.to_string()),275            }]),276        ),277        (278            format!("{metrics_port}/tcp"),279            Some(vec![PortBinding {280                host_ip: Some("127.0.0.1".to_owned()),281                host_port: Some(metrics_port.to_string()),282            }]),283        ),284    ]);285    let proxy_config = ContainerCreateBody {286        image: Some("basecamp/kamal-proxy:once-01".to_owned()),287        cmd: Some(vec![288            "kamal-proxy".to_owned(),289            "run".to_owned(),290            "--metrics-port".to_owned(),291            metrics_port.to_string(),292        ]),293        labels: Some(labels),294        exposed_ports: Some(vec![295            "80/tcp".to_owned(),296            "443/tcp".to_owned(),297            format!("{metrics_port}/tcp"),298        ]),299        host_config: Some(HostConfig {300            port_bindings: Some(port_bindings),301            restart_policy: Some(RestartPolicy {302                name: Some(RestartPolicyNameEnum::ALWAYS),303                maximum_retry_count: None,304            }),305            log_config: Some(HostConfigLogConfig {306                typ: Some("json-file".to_owned()),307                config: Some(HashMap::from([308                    ("max-size".to_owned(), "10m".to_owned()),309                    ("max-file".to_owned(), "1".to_owned()),310                ])),311            }),312            mounts: Some(vec![Mount {313                typ: Some(MountType::VOLUME),314                source: Some(proxy_name.to_owned()),315                target: Some("/home/kamal-proxy/.config/kamal-proxy".to_owned()),316                ..Default::default()317            }]),318            ..Default::default()319        }),320        networking_config: Some(NetworkingConfig {321            endpoints_config: Some(HashMap::from([(322                namespace.to_owned(),323                EndpointSettings::default(),324            )])),325        }),326        ..Default::default()327    };328    let create_options = CreateContainerOptionsBuilder::default()329        .name(proxy_name)330        .build();331    let proxy = docker332        .create_container(Some(create_options), proxy_config)333        .await334        .map_err(|error| error.to_string())?;335    Ok(proxy.id)336}337338async fn deploy_and_verify(339    docker: &Docker,340    namespace: &str,341    automatic_backup_directory: &Path,342) -> Result<String, String> {343    let deploy = ino(&[344        "once",345        "--namespace",346        namespace,347        "deploy",348        "ghcr.io/basecamp/writebook:latest",349        "--host",350        "localhost",351        "--disable-tls",352        "--env",353        "CUSTOM=port-test",354        "--smtp-server",355        "smtp.example.test",356        "--smtp-port",357        "587",358        "--smtp-username",359        "mailer",360        "--smtp-password",361        "test-password",362        "--smtp-from",363        "test@example.test",364        "--cpus",365        "1",366        "--memory",367        "2048",368        "--auto-update=false",369        "--backup-path",370        automatic_backup_directory.to_str().unwrap(),371        "--auto-backup",372    ])?;373    if !deploy.status.success() {374        return Err(format!(375            "deploy failed: {}",376            String::from_utf8_lossy(&deploy.stderr)377        ));378    }379    let list = ino(&["once", "--namespace", namespace, "list"])?;380    if !list.status.success()381        || !String::from_utf8_lossy(&list.stdout).contains("localhost (running)")382    {383        return Err(format!(384            "deployed app not listed: {}",385            String::from_utf8_lossy(&list.stdout)386        ));387    }388389    let containers = all_containers(docker).await?;390    let settings = settings_for_host(&containers, "localhost")391        .ok_or_else(|| "deployed app settings label not found".to_owned())?;392    if settings["env"]["CUSTOM"] != "port-test"393        || settings["smtp"]["server"] != "smtp.example.test"394        || settings["smtp"]["port"] != "587"395        || settings["resources"]["cpus"] != 1396        || settings["resources"]["memoryMB"] != 2048397        || settings["autoUpdate"] != false398        || settings["backup"]["path"] != automatic_backup_directory.to_string_lossy().as_ref()399        || settings["backup"]["autoBackup"] != true400    {401        return Err(format!(402            "deployment settings were not preserved: {settings}"403        ));404    }405    settings["name"]406        .as_str()407        .map(str::to_owned)408        .ok_or_else(|| "missing app name in settings".to_owned())409}410411async fn backup_and_remove(412    docker: &Docker,413    namespace: &str,414    backup_path: &Path,415    original_app_name: &str,416) -> Result<(), String> {417    let backup = ino(&[418        "once",419        "--namespace",420        namespace,421        "backup",422        "localhost",423        backup_path.to_str().unwrap(),424    ])?;425    if !backup.status.success() {426        return Err(format!(427            "backup failed: {}",428            String::from_utf8_lossy(&backup.stderr)429        ));430    }431    let archive_file = File::open(backup_path).map_err(|error| error.to_string())?;432    let mut archive = Archive::new(GzDecoder::new(archive_file));433    let mut entries = Vec::new();434    let mut archived_app_settings = None;435    for entry in archive.entries().map_err(|error| error.to_string())? {436        let mut entry = entry.map_err(|error| error.to_string())?;437        let path = entry438            .path()439            .map_err(|error| error.to_string())?440            .to_string_lossy()441            .into_owned();442        if path == "once.application.json" {443            let mut contents = String::new();444            entry445                .read_to_string(&mut contents)446                .map_err(|error| error.to_string())?;447            archived_app_settings = serde_json::from_str::<serde_json::Value>(&contents).ok();448        }449        entries.push(path);450    }451    if !entries.iter().any(|path| path == "once.volume.json")452        || !entries.iter().any(|path| path == "data")453        || archived_app_settings454            .as_ref()455            .is_none_or(|value| value["host"] != "localhost")456    {457        return Err(format!(458            "backup archive entries were incomplete: {entries:?}"459        ));460    }461462    let remove = ino(&["once", "--namespace", namespace, "remove", "localhost"])?;463    if !remove.status.success() {464        return Err(format!(465            "remove before restore failed: {}",466            String::from_utf8_lossy(&remove.stderr)467        ));468    }469    let old_volume = format!("{namespace}-app-{original_app_name}");470    if docker.inspect_volume(&old_volume).await.is_err() {471        return Err("removing app without --remove-data lost its volume".into());472    }473    docker474        .remove_volume(475            &old_volume,476            Some(RemoveVolumeOptionsBuilder::default().force(true).build()),477        )478        .await479        .map_err(|error| error.to_string())480}481482fn restore_and_update(namespace: &str, backup_path: &Path) -> Result<(), String> {483    let restore = ino(&[484        "once",485        "--namespace",486        namespace,487        "restore",488        backup_path.to_str().unwrap(),489    ])?;490    if !restore.status.success() {491        return Err(format!(492            "restore failed: {}",493            String::from_utf8_lossy(&restore.stderr)494        ));495    }496    let list = ino(&["once", "--namespace", namespace, "list"])?;497    if !list.status.success()498        || !String::from_utf8_lossy(&list.stdout).contains("localhost (running)")499    {500        return Err(format!(501            "restored app not listed: {}",502            String::from_utf8_lossy(&list.stdout)503        ));504    }505506    let update = ino(&[507        "once",508        "--namespace",509        namespace,510        "update",511        "localhost",512        "--smtp-from",513        "updated@example.test",514        "--auto-update=true",515    ])?;516    if !update.status.success() {517        return Err(format!(518            "update failed: {}",519            String::from_utf8_lossy(&update.stderr)520        ));521    }522    let updated_list = ino(&["once", "--namespace", namespace, "list"])?;523    if !updated_list.status.success()524        || !String::from_utf8_lossy(&updated_list.stdout).contains("localhost (running)")525    {526        return Err("updated app was not discoverable".into());527    }528    Ok(())529}530531async fn verify_updated_app(532    docker: &Docker,533    namespace: &str,534) -> Result<(String, serde_json::Value), String> {535    let containers = all_containers(docker).await?;536    let updated_settings = settings_for_host(&containers, "localhost")537        .ok_or_else(|| "updated settings label not found".to_owned())?;538    if updated_settings["smtp"]["from"] != "updated@example.test"539        || updated_settings["autoUpdate"] != true540    {541        return Err(format!(542            "update settings were not applied: {updated_settings}"543        ));544    }545    let updated_name = updated_settings["name"]546        .as_str()547        .ok_or_else(|| "updated app name missing".to_owned())?548        .to_owned();549    let app_prefix = format!("{namespace}-app-{updated_name}-");550    let app_containers = containers551        .iter()552        .filter(|container| {553            container.names.as_ref().is_some_and(|names| {554                names555                    .iter()556                    .any(|name| name.trim_start_matches('/').starts_with(&app_prefix))557            })558        })559        .count();560    if app_containers != 1 {561        return Err(format!("update left {app_containers} app containers"));562    }563    let updated_volume = format!("{namespace}-app-{updated_name}");564    if docker.inspect_volume(&updated_volume).await.is_err() {565        return Err("update did not preserve the app data volume".into());566    }567    Ok((updated_name, updated_settings))568}569570async fn rotate_keys(571    docker: &Docker,572    namespace: &str,573    updated_settings: &serde_json::Value,574) -> Result<(), String> {575    let old_vapid_public = updated_settings["keys"]["vapidPublicKey"]576        .as_str()577        .unwrap_or_default()578        .to_owned();579    let reset = ino(&[580        "once",581        "--namespace",582        namespace,583        "keys",584        "reset",585        "localhost",586        "--vapid",587    ])?;588    if !reset.status.success() {589        return Err(format!(590            "key reset failed: {}",591            String::from_utf8_lossy(&reset.stderr)592        ));593    }594    let reset_settings = app_settings_for_host(docker, "localhost").await?;595    if reset_settings["keys"]["vapidPublicKey"] == old_vapid_public {596        return Err("VAPID reset did not change the public key".into());597    }598    let vapid_private = reset_settings["keys"]["vapidPrivateKey"]599        .as_str()600        .ok_or_else(|| "reset VAPID private key missing".to_owned())?;601    let set_vapid_arg = format!("--vapid={vapid_private}");602    let set = ino(&[603        "once",604        "--namespace",605        namespace,606        "keys",607        "set",608        "localhost",609        "--secret-key-base",610        "replacement-secret-key-base",611        &set_vapid_arg,612    ])?;613    if !set.status.success() {614        return Err(format!(615            "key set failed: {}",616            String::from_utf8_lossy(&set.stderr)617        ));618    }619    let set_settings = app_settings_for_host(docker, "localhost").await?;620    if set_settings["keys"]["secretKeyBase"] != "replacement-secret-key-base"621        || set_settings["keys"]["vapidPrivateKey"] != vapid_private622    {623        return Err("key set did not persist the requested key values".into());624    }625    Ok(())626}627628async fn verify_background_checks(629    docker: &Docker,630    namespace: &str,631    proxy_name: &str,632    automatic_backup_directory: &Path,633    updated_name: &str,634) -> Result<(), String> {635    std::fs::create_dir_all(automatic_backup_directory).map_err(|error| error.to_string())?;636    let background = ino(&[637        "once",638        "--namespace",639        namespace,640        "background",641        "run",642        "--once",643        "--no-self-update",644    ])?;645    if !background.status.success() {646        return Err(format!(647            "background check failed: {}",648            String::from_utf8_lossy(&background.stderr)649        ));650    }651    if std::fs::read_dir(automatic_backup_directory)652        .map_err(|error| error.to_string())?653        .next()654        .is_none()655    {656        return Err("background check did not create an automatic backup".into());657    }658659    let state_options = bollard::query_parameters::DownloadFromContainerOptionsBuilder::default()660        .path("/home/kamal-proxy/.config/kamal-proxy/once-state.json")661        .build();662    let mut state_stream = docker.download_from_container(proxy_name, Some(state_options));663    let mut state_tar = Vec::new();664    while let Some(chunk) = state_stream.next().await {665        state_tar.extend_from_slice(&chunk.map_err(|error| error.to_string())?);666    }667    let mut state_archive = tar::Archive::new(Cursor::new(state_tar));668    let mut state_json = String::new();669    for entry in state_archive.entries().map_err(|error| error.to_string())? {670        let mut entry = entry.map_err(|error| error.to_string())?;671        if entry672            .path()673            .map_err(|error| error.to_string())?674            .to_string_lossy()675            == "once-state.json"676        {677            entry678                .read_to_string(&mut state_json)679                .map_err(|error| error.to_string())?;680        }681    }682    let state: serde_json::Value =683        serde_json::from_str(&state_json).map_err(|error| error.to_string())?;684    if state["apps"][updated_name]["lastBackup"]["at"]685        .as_str()686        .unwrap_or_default()687        .is_empty()688        || state["apps"][updated_name]["lastUpdate"]["at"]689            .as_str()690            .unwrap_or_default()691            .is_empty()692    {693        return Err(format!(694            "operation history missing from Once state: {state}"695        ));696    }697    Ok(())698}699700async fn teardown_and_verify(docker: &Docker, namespace: &str) -> Result<(), String> {701    let teardown = ino(&[702        "once",703        "--namespace",704        namespace,705        "teardown",706        "--remove-data",707    ])?;708    if !teardown.status.success() {709        return Err(format!(710            "teardown failed: {}",711            String::from_utf8_lossy(&teardown.stderr)712        ));713    }714    let containers = all_containers(docker).await?;715    if containers.iter().any(|container| {716        container.names.as_ref().is_some_and(|names| {717            names.iter().any(|name| {718                name.trim_start_matches('/')719                    .starts_with(&format!("{namespace}-"))720            })721        })722    }) {723        return Err("teardown left namespace containers behind".into());724    }725    let volumes = docker726        .list_volumes(Some(727            bollard::query_parameters::ListVolumesOptionsBuilder::default().build(),728        ))729        .await730        .map_err(|error| error.to_string())?;731    if volumes732        .volumes733        .unwrap_or_default()734        .iter()735        .any(|volume| volume.name.starts_with(&format!("{namespace}-")))736    {737        return Err("teardown with --remove-data left namespace volumes behind".into());738    }739    Ok(())740}741742async fn cleanup_namespace(docker: &Docker, namespace: &str) {743    let containers = docker744        .list_containers(Some(745            bollard::query_parameters::ListContainersOptionsBuilder::default()746                .all(true)747                .build(),748        ))749        .await750        .unwrap_or_default();751    for container in containers {752        let Some(names) = container.names else {753            continue;754        };755        if names.iter().any(|name| {756            name.trim_start_matches('/')757                .starts_with(&format!("{namespace}-"))758        }) && let Some(id) = container.id759        {760            let _ = docker761                .remove_container(762                    &id,763                    Some(RemoveContainerOptionsBuilder::default().force(true).build()),764                )765                .await;766        }767    }768769    if let Ok(volumes) = docker770        .list_volumes(Some(771            bollard::query_parameters::ListVolumesOptionsBuilder::default().build(),772        ))773        .await774    {775        for volume in volumes.volumes.unwrap_or_default() {776            if volume.name.starts_with(&format!("{namespace}-")) {777                let _ = docker778                    .remove_volume(779                        &volume.name,780                        Some(RemoveVolumeOptionsBuilder::default().force(true).build()),781                    )782                    .await;783            }784        }785    }786    let _ = docker.remove_network(namespace).await;787}788789async fn all_containers(docker: &Docker) -> Result<Vec<ContainerSummary>, String> {790    docker791        .list_containers(Some(792            bollard::query_parameters::ListContainersOptionsBuilder::default()793                .all(true)794                .build(),795        ))796        .await797        .map_err(|error| error.to_string())798}799800fn settings_for_host(containers: &[ContainerSummary], host: &str) -> Option<serde_json::Value> {801    containers802        .iter()803        .filter_map(|container| container.labels.as_ref())804        .find_map(|labels| {805            labels806                .get("once")807                .and_then(|label| serde_json::from_str::<serde_json::Value>(label).ok())808                .filter(|value| value["host"] == host)809        })810}811812async fn app_settings_for_host(docker: &Docker, host: &str) -> Result<serde_json::Value, String> {813    let containers = all_containers(docker).await?;814    settings_for_host(&containers, host)815        .ok_or_else(|| format!("no app metadata found for host {host}"))816}817818fn ino(args: &[&str]) -> Result<std::process::Output, String> {819    Command::new(env!("CARGO_BIN_EXE_ino"))820        .args(args)821        .stdin(Stdio::null())822        .output()823        .map_err(|error| error.to_string())824}