Skip to content
1        10        20        30        40        50        60        70        80        90        100       110       120       130       140       150       1601//! The Deploy tab's Git flow. The console clones the repository on this2//! machine and runs `ino` in the clone, with `--events json`, to release and3//! deploy it. The release is the CLI's own code, so a terminal and the console4//! do the same steps. This module starts that process and turns its events5//! into job events the page shows.67use std::{8    collections::BTreeMap,9    fs,10    io::{BufReader, Read, Write},11    path::{Path, PathBuf},12    process::{Command, Stdio},13    sync::PoisonError,14    thread,15};1617use actix_web::{HttpResponse, http::StatusCode, web};18use serde::{Deserialize, de::DeserializeOwned};19use serde_json::{Value, json};2021use super::{22    ConsoleState,23    jobs::{Busy, Job},24    service,25};26use crate::commands::{new, release};2728const REPOS_DIR: &str = "repos";29const ERROR_TAIL_LINES: usize = 5;3031#[derive(Deserialize)]32struct CheckBody {33    url: String,34}3536#[derive(Deserialize)]37#[serde(rename_all = "camelCase")]38struct ReleaseBody {39    url: String,40    action: String,41    #[serde(default)]42    env: BTreeMap<String, String>,43}4445#[derive(Deserialize)]46pub struct AfterQuery {47    #[serde(default)]48    after: usize,49}5051/// The fields of a server app summary that the console reads.52#[derive(Deserialize)]53#[serde(rename_all = "camelCase")]54struct AppSummary {55    host: String,56    image: String,57    running: bool,58}5960/// The server and key the console manages, copied for a job thread.61#[derive(Clone)]62struct Server {63    origin: String,64    api_key: String,65    client: reqwest::Client,66}6768/// A deploy target from the Dockerfile's ino.deploy labels, checked against the69/// server this console manages.70struct Target {71    server: String,72    host: String,73}7475/// What a clone holds and what its Dockerfile declares.76struct Inspection {77    folder: PathBuf,78    branch: String,79    commit: String,80    subject: String,81    version: String,82    image: String,83    labels: Option<release::DeploymentTarget>,84}8586/// An error the page shows for a request: the HTTP status, a stable code and a87/// message.88struct Failure {89    status: StatusCode,90    code: &'static str,91    message: String,92}9394impl Failure {95    fn new(status: StatusCode, code: &'static str, message: impl Into<String>) -> Self {96        Self {97            status,98            code,99            message: message.into(),100        }101    }102103    fn into_response(self) -> HttpResponse {104        HttpResponse::build(self.status).json(json!({105            "error": { "code": self.code, "message": self.message }106        }))107    }108}109110pub async fn start_check(state: web::Data<ConsoleState>, body: web::Bytes) -> HttpResponse {111    let request: CheckBody = match parse(&body) {112        Ok(request) => request,113        Err(failure) => return failure.into_response(),114    };115    let url = match validate_url(&request.url) {116        Ok(url) => url,117        Err(failure) => return failure.into_response(),118    };119    let server = server_of(&state);120    start(&state, "check", move |job| check_job(job, &url, &server))121}122123pub async fn start_release(state: web::Data<ConsoleState>, body: web::Bytes) -> HttpResponse {124    let request: ReleaseBody = match parse(&body) {125        Ok(request) => request,126        Err(failure) => return failure.into_response(),127    };128    let url = match validate_url(&request.url) {129        Ok(url) => url,130        Err(failure) => return failure.into_response(),131    };132    if request.action != "create" && request.action != "update" {133        return Failure::new(134            StatusCode::BAD_REQUEST,135            "invalid_action",136            "the action must be create or update",137        )138        .into_response();139    }140    let environment = match environment_file(&request.env) {141        Ok(text) => text,142        Err(failure) => return failure.into_response(),143    };144    let action = request.action;145    let server = server_of(&state);146    start(&state, "release", move |job| {147        release_job(job, &url, &action, &environment, &server)148    })149}150151/// The job's status and the events after the index the page already has.152pub async fn status(153    state: web::Data<ConsoleState>,154    path: web::Path<String>,155    query: web::Query<AfterQuery>,156) -> HttpResponse {157    let jobs = state.jobs.lock().unwrap_or_else(PoisonError::into_inner);158    match jobs.find(&path) {159        Some(job) => HttpResponse::Ok().json(job.snapshot(query.after)),160        None => Failure::new(161            StatusCode::NOT_FOUND,162            "job_not_found",163            "that job is not known",164        )165        .into_response(),166    }167}168169fn start<F>(state: &ConsoleState, kind: &'static str, work: F) -> HttpResponse170where171    F: FnOnce(&Job) -> Result<Value, String> + Send + 'static,172{173    let mut jobs = state.jobs.lock().unwrap_or_else(PoisonError::into_inner);174    match jobs.start(kind, work) {175        Ok(id) => HttpResponse::Accepted().json(json!({ "id": id })),176        Err(Busy) => Failure::new(177            StatusCode::CONFLICT,178            "busy",179            "another deploy step is still running",180        )181        .into_response(),182    }183}184185fn server_of(state: &ConsoleState) -> Server {186    Server {187        origin: state.origin().to_owned(),188        api_key: state.api_key.clone(),189        client: state.client.clone(),190    }191}192193fn parse<T: DeserializeOwned>(body: &[u8]) -> Result<T, Failure> {194    serde_json::from_slice(body).map_err(|error| {195        Failure::new(196            StatusCode::BAD_REQUEST,197            "invalid_request",198            format!("reading the request: {error}"),199        )200    })201}202203/// Accepts the URL forms git clones over the network. Other transports, such204/// as `ext::` and `file://`, can run local commands, so they are refused, and205/// a leading dash would be read as a git option.206fn validate_url(raw: &str) -> Result<String, Failure> {207    let url = raw.trim();208    let invalid = |message: &str| Failure::new(StatusCode::BAD_REQUEST, "invalid_url", message);209    if url.is_empty() {210        return Err(invalid("enter the repository URL"));211    }212    if url.starts_with('-')213        || url214            .chars()215            .any(|character| character.is_whitespace() || character.is_control())216    {217        return Err(invalid(218            "the repository URL must not contain spaces or start with a dash",219        ));220    }221    if ["https://", "http://", "ssh://"]222        .iter()223        .any(|scheme| url.starts_with(scheme))224        || is_scp_like(url)225    {226        return Ok(url.to_owned());227    }228    Err(invalid(229        "use an https://, http://, ssh:// or user@host:path repository URL",230    ))231}232233/// The `user@host:path` form git accepts for SSH.234fn is_scp_like(url: &str) -> bool {235    let Some((user_host, path)) = url.split_once(':') else {236        return false;237    };238    let Some((user, host)) = user_host.split_once('@') else {239        return false;240    };241    !user.is_empty() && !host.is_empty() && !path.is_empty()242}243244/// Checks the environment the page sends and turns it into the `KEY=VALUE`245/// lines the release reads from its standard input.246fn environment_file(env: &BTreeMap<String, String>) -> Result<String, Failure> {247    let mut text = String::new();248    for (name, value) in env {249        let mut characters = name.chars();250        let valid_name = characters251            .next()252            .is_some_and(|first| first.is_ascii_alphabetic() || first == '_')253            && characters.all(|character| character.is_ascii_alphanumeric() || character == '_');254        if !valid_name {255            return Err(Failure::new(256                StatusCode::BAD_REQUEST,257                "invalid_env",258                format!("{name:?} is not a valid variable name"),259            ));260        }261        if value.contains(['\n', '\r', '\0']) {262            return Err(Failure::new(263                StatusCode::BAD_REQUEST,264                "invalid_env",265                format!("the value of {name} must be on one line"),266            ));267        }268        text.push_str(name);269        text.push('=');270        text.push_str(value);271        text.push('\n');272    }273    Ok(text)274}275276/// Folder name per URL: its readable part, then a hash so two URLs never share a clone.277fn folder_name(url: &str) -> String {278    let mut readable = String::new();279    for character in url.chars() {280        if character.is_ascii_alphanumeric() {281            readable.push(character.to_ascii_lowercase());282        } else if !readable.ends_with('-') {283            readable.push('-');284        }285    }286    let readable: String = readable.trim_matches('-').chars().take(48).collect();287    format!("{readable}-{:016x}", fnv1a(url))288}289290fn fnv1a(text: &str) -> u64 {291    text.bytes().fold(0xcbf2_9ce4_8422_2325, |hash, byte| {292        (hash ^ u64::from(byte)).wrapping_mul(0x0000_0100_0000_01b3)293    })294}295296/// The tag of an image reference. Digests are ignored, and a missing tag means297/// `latest`, as Docker reads it.298fn image_tag(reference: &str) -> &str {299    let name = reference.split('@').next().unwrap_or(reference);300    let leaf_start = name.rfind('/').map_or(0, |slash| slash + 1);301    match name[leaf_start..].find(':') {302        Some(colon) => &name[leaf_start + colon + 1..],303        None => "latest",304    }305}306307fn check_job(job: &Job, url: &str, server: &Server) -> Result<Value, String> {308    let inspection = clone_and_inspect(job, url)?;309    let (target, problem) = match deploy_target(inspection.labels.as_ref(), &server.origin) {310        Ok(target) => (Some(target), None),311        Err(problem) => (None, Some(problem)),312    };313    let deployed = match &target {314        Some(target) => {315            job.step(&format!("Checking {} on {}", target.host, target.server));316            find_deployed(server, &target.host)?317        }318        None => None,319    };320    Ok(json!({321        "folder": inspection.folder,322        "branch": inspection.branch,323        "commit": inspection.commit,324        "subject": inspection.subject,325        "version": inspection.version,326        "image": inspection.image,327        "target": target.map(|target| json!({ "server": target.server, "host": target.host })),328        "problem": problem,329        "deployed": deployed,330    }))331}332333fn release_job(334    job: &Job,335    url: &str,336    action: &str,337    environment: &str,338    server: &Server,339) -> Result<Value, String> {340    let inspection = clone_and_inspect(job, url)?;341    let target = deploy_target(inspection.labels.as_ref(), &server.origin)?;342343    let mut command = Command::new(344        std::env::current_exe().map_err(|error| format!("finding the ino program: {error}"))?,345    );346    command347        .current_dir(&inspection.folder)348        .env("INO_API_KEY", &server.api_key)349        .args(["--events", "json"]);350    let stdin = if action == "create" {351        command.args(["once", "remote", "create", "--env-file", "-", "--yes"]);352        Stdio::piped()353    } else {354        command.arg("auto");355        Stdio::null()356    };357    job.step(&format!(358        "{} {} on {}",359        if action == "create" {360            "Creating"361        } else {362            "Releasing and deploying"363        },364        target.host,365        target.server366    ));367    run_ino(job, command.stdin(stdin), environment, action == "create")368}369370/// Runs `ino` and turns its JSON events into job events. The job succeeds371/// only when the process exits cleanly after its `done` event.372fn run_ino(373    job: &Job,374    command: &mut Command,375    environment: &str,376    send_environment: bool,377) -> Result<Value, String> {378    command.stdout(Stdio::piped()).stderr(Stdio::piped());379    let mut child = command380        .spawn()381        .map_err(|error| format!("starting ino: {error}"))?;382    if send_environment && let Some(mut stdin) = child.stdin.take() {383        let _ = stdin.write_all(environment.as_bytes());384    }385    let stdout = child.stdout.take();386    let stderr = child.stderr.take();387388    let mut done: Option<Value> = None;389    let mut failure: Option<String> = None;390    thread::scope(|scope| {391        if let Some(stderr) = stderr {392            scope.spawn(move || {393                read_lines(stderr, |line| job.output("stderr", line));394            });395        }396        if let Some(stdout) = stdout {397            read_lines(stdout, |line| match serde_json::from_str::<Value>(line) {398                Ok(event) if event["type"].is_string() => {399                    match event["type"].as_str() {400                        Some("done") => done = Some(event.clone()),401                        Some("error") => {402                            failure = event["message"].as_str().map(str::to_owned);403                        }404                        _ => {}405                    }406                    job.push(event);407                }408                _ => job.output("stdout", line),409            });410        }411    });412413    let status = child414        .wait()415        .map_err(|error| format!("waiting for ino: {error}"))?;416    match done {417        Some(done) if status.success() => Ok(done),418        _ => Err(failure.unwrap_or_else(|| format!("ino stopped with {status}"))),419    }420}421422/// Clones or updates the repository, then reads what the clone holds.423fn clone_and_inspect(job: &Job, url: &str) -> Result<Inspection, String> {424    job.step("Cloning or updating the repository");425    let root = service::state_dir()426        .map(|state| state.join(REPOS_DIR))427        .map_err(|error| error.to_string())?;428    let folder = root.join(folder_name(url));429    if folder.join(".git").is_dir() {430        run_git(431            job,432            git(Some(&folder)).args(["fetch", "--progress", "--prune", "origin"]),433            "fetching the repository",434        )?;435        run_git(436            job,437            git(Some(&folder)).args(["remote", "set-head", "origin", "--auto"]),438            "finding the default branch",439        )?;440        run_git(441            job,442            git(Some(&folder)).args(["reset", "--hard", "origin/HEAD"]),443            "resetting the clone",444        )?;445        run_git(446            job,447            git(Some(&folder)).args(["clean", "-ffdx"]),448            "cleaning the clone",449        )?;450    } else {451        if folder.exists() {452            fs::remove_dir_all(&folder)453                .map_err(|error| format!("clearing an incomplete clone: {error}"))?;454        }455        fs::create_dir_all(&root)456            .map_err(|error| format!("creating the repository folder: {error}"))?;457        let mut command = git(None);458        command459            .args(["clone", "--progress", "--", url])460            .arg(&folder);461        run_git(job, &mut command, "cloning the repository")?;462    }463    inspect_folder(&folder)464}465466/// Reads the Dockerfile's labels and the commit and version of the clone.467fn inspect_folder(folder: &Path) -> Result<Inspection, String> {468    let dockerfile = fs::read_to_string(folder.join("Dockerfile"))469        .map_err(|error| format!("the repository needs a Dockerfile at its root: {error}"))?;470    let metadata = release::parse_image_metadata(&dockerfile).map_err(|error| error.to_string())?;471472    let repository =473        git2::Repository::open(folder).map_err(|error| format!("reading the clone: {error}"))?;474    let head = repository475        .head()476        .map_err(|error| format!("reading HEAD: {error}"))?;477    let branch = head.shorthand().unwrap_or("HEAD").to_owned();478    let commit = head479        .peel_to_commit()480        .map_err(|error| format!("resolving HEAD: {error}"))?;481    let subject = String::from_utf8_lossy(commit.summary_bytes().unwrap_or_default()).into_owned();482    let info = new::version_info_in(folder).map_err(|error| error.to_string())?;483484    Ok(Inspection {485        folder: folder.to_path_buf(),486        branch,487        commit: info.commit_hash,488        subject,489        version: info.semver.to_string(),490        image: metadata.remote,491        labels: metadata.deployment,492    })493}494495/// The labels' target, if it is on the server this console manages.496fn deploy_target(497    labels: Option<&release::DeploymentTarget>,498    origin: &str,499) -> Result<Target, String> {500    let Some(labels) = labels else {501        return Err(502            "the Dockerfile needs ino.deploy.server and ino.deploy.host labels to deploy this repository"503                .to_owned(),504        );505    };506    let server = labels.server.as_str().trim_end_matches('/').to_owned();507    if server != origin {508        return Err(format!(509            "the Dockerfile deploys to {server}, but this console manages {origin}"510        ));511    }512    Ok(Target {513        server,514        host: labels.host.clone(),515    })516}517518/// The app on the server with this host: its tag and whether it runs.519fn find_deployed(server: &Server, host: &str) -> Result<Option<Value>, String> {520    let runtime = tokio::runtime::Builder::new_current_thread()521        .enable_all()522        .build()523        .map_err(|error| format!("starting a request: {error}"))?;524    let apps: Vec<AppSummary> = runtime.block_on(async {525        let response = server526            .client527            .get(format!("{}/api/v1/apps", server.origin))528            .bearer_auth(&server.api_key)529            .send()530            .await531            .map_err(|_| "the Ino server could not be reached".to_owned())?;532        if !response.status().is_success() {533            return Err(format!(534                "the Ino server rejected the app list ({})",535                response.status()536            ));537        }538        response539            .json()540            .await541            .map_err(|error| format!("reading the app list: {error}"))542    })?;543    Ok(apps544        .into_iter()545        .find(|app| app.host == host)546        .map(|app| json!({ "version": image_tag(&app.image), "running": app.running })))547}548549fn git(folder: Option<&Path>) -> Command {550    let mut command = Command::new("git");551    if let Some(folder) = folder {552        command.arg("-C").arg(folder);553    }554    command.env("GIT_TERMINAL_PROMPT", "0").stdin(Stdio::null());555    command556}557558/// Runs git, sending its progress to the job as it arrives. A failure reports559/// git's last error lines.560fn run_git(job: &Job, command: &mut Command, action: &str) -> Result<(), String> {561    let mut child = command562        .stdout(Stdio::null())563        .stderr(Stdio::piped())564        .spawn()565        .map_err(|error| format!("running git: {error}"))?;566    let lines = match child.stderr.take() {567        Some(stderr) => read_lines(stderr, |line| job.output("stderr", line)),568        None => Vec::new(),569    };570    let status = child571        .wait()572        .map_err(|error| format!("waiting for git: {error}"))?;573    if status.success() {574        return Ok(());575    }576    let tail = lines[lines.len().saturating_sub(ERROR_TAIL_LINES)..].join("\n");577    Err(format!("{action}:\n{tail}"))578}579580/// Calls `each` with every line of `source`. Carriage returns end a line too,581/// so progress that rewrites one line still arrives as it is written. Returns582/// all lines.583fn read_lines(source: impl Read, mut each: impl FnMut(&str)) -> Vec<String> {584    let mut lines = Vec::new();585    let mut current = Vec::new();586    for byte in BufReader::new(source).bytes() {587        let Ok(byte) = byte else { break };588        if byte == b'\n' || byte == b'\r' {589            if !current.is_empty() {590                let line = String::from_utf8_lossy(&current).into_owned();591                each(&line);592                lines.push(line);593                current.clear();594            }595        } else {596            current.push(byte);597        }598    }599    if !current.is_empty() {600        let line = String::from_utf8_lossy(&current).into_owned();601        each(&line);602        lines.push(line);603    }604    lines605}606607#[cfg(test)]608mod tests {609    use std::{collections::BTreeMap, path::Path, process::Command};610611    use super::{612        deploy_target, environment_file, folder_name, image_tag, inspect_folder, validate_url,613    };614615    fn git_in(folder: &Path, args: &[&str]) {616        let status = Command::new("git")617            .arg("-C")618            .arg(folder)619            .args(args)620            .env("GIT_AUTHOR_NAME", "test")621            .env("GIT_AUTHOR_EMAIL", "test@example.com")622            .env("GIT_COMMITTER_NAME", "test")623            .env("GIT_COMMITTER_EMAIL", "test@example.com")624            .status()625            .unwrap();626        assert!(status.success(), "git {args:?} failed");627    }628629    /// Writes a project with its labels and commits it. The note changes with630    /// each message so every call makes a new commit.631    fn commit_project(folder: &Path, version: &str, message: &str) {632        std::fs::write(633            folder.join("Cargo.toml"),634            format!("[package]\nname = \"sample\"\nversion = \"{version}\"\n"),635        )636        .unwrap();637        std::fs::write(638            folder.join("Dockerfile"),639            "FROM scratch\nLABEL remote=\"example/sample\" \\\n      ino.deploy.server=\"http://127.0.0.1:9911\" \\\n      ino.deploy.host=\"sample.example.com\"\n",640        )641        .unwrap();642        std::fs::write(folder.join("NOTES.md"), format!("{message}\n")).unwrap();643        git_in(folder, &["add", "."]);644        git_in(folder, &["commit", "-q", "-m", message]);645    }646647    #[test]648    fn accepts_network_git_urls_and_refuses_other_transports() {649        for url in [650            "https://git.example.com/team/app.git",651            "http://git.example.com/team/app",652            "ssh://git@git.example.com/team/app.git",653            "git@git.example.com:team/app.git",654            "  https://git.example.com/team/app  ",655        ] {656            assert!(validate_url(url).is_ok(), "{url} should be accepted");657        }658        for url in [659            "",660            "-oProxyCommand=evil",661            "file:///srv/git/app",662            "ext::sh -c touch",663            "https://git.example.com/a b",664            "git@git.example.com",665        ] {666            assert!(validate_url(url).is_err(), "{url} should be refused");667        }668    }669670    #[test]671    fn folder_names_are_stable_and_distinct_per_url() {672        let first = folder_name("https://git.example.com/team/app.git");673        assert_eq!(first, folder_name("https://git.example.com/team/app.git"));674        assert!(first.starts_with("https-git-example-com-team-app-git-"));675        assert_ne!(first, folder_name("https://git.example.com/team/app"));676        assert!(!first.contains('/'));677    }678679    #[test]680    fn reads_the_tag_of_an_image_reference() {681        assert_eq!(image_tag("silenloc/ino:1.2.1"), "1.2.1");682        assert_eq!(image_tag("silenloc/ino"), "latest");683        assert_eq!(image_tag("registry.example.com:5000/team/app"), "latest");684        assert_eq!(image_tag("team/app:2@sha256:abc"), "2");685    }686687    #[test]688    fn environment_lines_are_checked_before_they_reach_the_release() {689        let mut env = BTreeMap::new();690        env.insert("RAILS_ENV".to_owned(), "production".to_owned());691        env.insert("_TOKEN".to_owned(), "a=b".to_owned());692        assert_eq!(693            environment_file(&env).ok().unwrap(),694            "RAILS_ENV=production\n_TOKEN=a=b\n"695        );696697        env.insert("1BAD".to_owned(), "x".to_owned());698        assert!(environment_file(&env).is_err());699        env.remove("1BAD");700        env.insert("OK".to_owned(), "line\nbreak".to_owned());701        assert!(environment_file(&env).is_err());702    }703704    #[test]705    fn labels_must_name_this_console_server() {706        let labels = crate::commands::release::DeploymentTarget {707            server: reqwest::Url::parse("https://ino.example.com").unwrap(),708            host: "app.example.com".to_owned(),709        };710        assert!(deploy_target(Some(&labels), "https://ino.example.com").is_ok());711        assert!(deploy_target(Some(&labels), "https://other.example.com").is_err());712        assert!(deploy_target(None, "https://ino.example.com").is_err());713    }714715    #[test]716    fn clones_then_updates_a_repository_and_reads_its_version() {717        let source = tempfile::tempdir().unwrap();718        let root = tempfile::tempdir().unwrap();719        git_in(source.path(), &["init", "-q"]);720        commit_project(source.path(), "0.3.0", "feat: first");721722        let url = source.path().to_str().unwrap();723        let folder = root.path().join(folder_name(url));724        std::fs::create_dir_all(root.path()).unwrap();725        let output = Command::new("git")726            .args(["clone", "-q", "--", url])727            .arg(&folder)728            .status()729            .unwrap();730        assert!(output.success());731        let first = inspect_folder(&folder).unwrap();732        assert_eq!(first.version, "0.3.0");733        assert_eq!(first.subject, "feat: first");734        assert_eq!(first.image, "example/sample");735        assert!(first.labels.is_some());736737        // Without a version bump the version stays, as `ino release` leaves it.738        commit_project(source.path(), "0.3.0", "fix: second");739        git_in(&folder, &["fetch", "-q", "origin"]);740        git_in(&folder, &["reset", "-q", "--hard", "origin/HEAD"]);741        let second = inspect_folder(&folder).unwrap();742        assert_ne!(second.commit, first.commit);743        assert_eq!(second.version, "0.3.0");744    }745}