LINE1 10 20 30 40 50 60 70 80 90123456789use std::path::{Path, PathBuf};1011use futures_util::StreamExt;12use tokio::io::AsyncWriteExt;131415const FILES: [&str; 4] = [16 "config.json",17 "encoder_config/config.json",18 "tokenizer.json",19 "model.safetensors",20];212223pub(crate) async fn ensure(24 model_dir: &Path,25 repo: &str,26 endpoint: &str,27 token: Option<&str>,28) -> Result<(), String> {29 std::fs::create_dir_all(model_dir.join("encoder_config"))30 .map_err(|error| format!("cannot create {}: {error}", model_dir.display()))?;3132 let client = reqwest::Client::builder()33 .user_agent(concat!("local-decisions/", env!("CARGO_PKG_VERSION")))34 .build()35 .map_err(|error| format!("cannot build HTTP client: {error}"))?;3637 for file in FILES {38 let destination = model_dir.join(file);39 if is_present(&destination) {40 log::debug!("{} is already present", destination.display());41 continue;42 }43 log::info!("downloading {repo}/{file}");44 download(&client, repo, endpoint, token, file, &destination).await?;45 }46 Ok(())47}4849fn is_present(path: &Path) -> bool {50 std::fs::metadata(path).is_ok_and(|meta| meta.len() > 0)51}5253async fn download(54 client: &reqwest::Client,55 repo: &str,56 endpoint: &str,57 token: Option<&str>,58 file: &str,59 destination: &Path,60) -> Result<(), String> {61 const ATTEMPTS: usize = 5;6263 let partial = partial_path(destination);64 let mut last_error = String::new();65 for attempt in 1..=ATTEMPTS {66 match transfer(client, repo, endpoint, token, file, destination, &partial).await {67 Ok(bytes) => {68 log::info!("stored {file} ({bytes} bytes)");69 return Ok(());70 }71 Err(error) => {72 log::warn!("download of {file} failed ({attempt}/{ATTEMPTS}): {error}");73 last_error = error;74 }75 }76 }77 Err(last_error)78}798081async fn transfer(82 client: &reqwest::Client,83 repo: &str,84 endpoint: &str,85 token: Option<&str>,86 file: &str,87 destination: &Path,88 partial: &Path,89) -> Result<u64, String> {90 let url = format!("{endpoint}/{repo}/resolve/main/{file}");91 let resumed = std::fs::metadata(partial).map_or(0, |meta| meta.len());92 let mut request = client.get(&url);93 if let Some(token) = token {94 request = request.bearer_auth(token);95 }96 if resumed > 0 {97 request = request.header(reqwest::header::RANGE, format!("bytes={resumed}-"));98 }99100 let response = request101 .send()102 .await103 .map_err(|error| format!("cannot fetch {url}: {error}"))?;104 if !response.status().is_success() {105 return Err(format!("{url} returned {}", response.status()));106 }107 let appending = response.status() == reqwest::StatusCode::PARTIAL_CONTENT;108109 let mut file_out = open(partial, appending)?;110 let mut written = if appending { resumed } else { 0 };111 let mut stream = response.bytes_stream();112 while let Some(chunk) = stream.next().await {113 let chunk = chunk.map_err(|error| format!("transfer of {file} failed: {error}"))?;114 file_out115 .write_all(&chunk)116 .await117 .map_err(|error| format!("cannot write {}: {error}", partial.display()))?;118 written += chunk.len() as u64;119 }120 file_out121 .flush()122 .await123 .map_err(|error| format!("cannot flush {}: {error}", partial.display()))?;124 drop(file_out);125126 if written == 0 {127 return Err(format!("{file} downloaded empty"));128 }129 tokio::fs::rename(partial, destination)130 .await131 .map_err(|error| format!("cannot move {} into place: {error}", partial.display()))?;132 Ok(written)133}134135fn open(path: &Path, appending: bool) -> Result<tokio::fs::File, String> {136 let mut options = std::fs::OpenOptions::new();137 options.write(true).create(true);138 if appending {139 options.append(true);140 } else {141 options.truncate(true);142 }143 Ok(tokio::fs::File::from_std(options.open(path).map_err(144 |error| format!("cannot open {}: {error}", path.display()),145 )?))146}147148fn partial_path(destination: &Path) -> PathBuf {149 let mut name = destination.as_os_str().to_owned();150 name.push(".part");151 PathBuf::from(name)152}