From 897bbeb198b5ef86d1c3a12a859ff9de46e0e975 Mon Sep 17 00:00:00 2001 From: Nico Fricke Date: Wed, 15 Jan 2025 09:13:00 +0100 Subject: [PATCH] implement fileupload --- Cargo.lock | 148 +++++++++++++++++++++++++ casket-backend/Cargo.toml | 12 +- casket-backend/casket.toml | 5 +- casket-backend/src/config.rs | 6 + casket-backend/src/errors.rs | 4 + casket-backend/src/files/mod.rs | 181 +++++++++++++++++++++++++++++++ casket-backend/src/main.rs | 12 +- casket-backend/src/routes/mod.rs | 9 +- 8 files changed, 367 insertions(+), 10 deletions(-) create mode 100644 casket-backend/src/errors.rs create mode 100644 casket-backend/src/files/mod.rs diff --git a/Cargo.lock b/Cargo.lock index 9ed286c..4a4ac9c 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -39,6 +39,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "6d6fd624c75e18b3b4c6b9caf42b1afe24437daaee904069137d8bab077be8b8" dependencies = [ "axum-core", + "axum-macros", "bytes", "form_urlencoded", "futures-util", @@ -51,6 +52,7 @@ dependencies = [ "matchit", "memchr", "mime", + "multer", "percent-encoding", "pin-project-lite", "rustversion", @@ -86,6 +88,17 @@ dependencies = [ "tracing", ] +[[package]] +name = "axum-macros" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "604fde5e028fea851ce1d8570bbdc034bec850d157f7569d10f347d06808c05c" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + [[package]] name = "backtrace" version = "0.3.74" @@ -125,8 +138,12 @@ version = "0.1.0" dependencies = [ "axum", "figment", + "futures", + "futures-util", + "problem_details", "serde", "tokio", + "tokio-stream", "tracing", "tracing-log", "tracing-subscriber", @@ -138,6 +155,15 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd" +[[package]] +name = "encoding_rs" +version = "0.8.35" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75030f3c4f45dafd7586dd6780965a8c7e8e285a5ecb86713e63a79c5b2766f3" +dependencies = [ + "cfg-if", +] + [[package]] name = "equivalent" version = "1.0.1" @@ -173,6 +199,21 @@ dependencies = [ "percent-encoding", ] +[[package]] +name = "futures" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876" +dependencies = [ + "futures-channel", + "futures-core", + "futures-executor", + "futures-io", + "futures-sink", + "futures-task", + "futures-util", +] + [[package]] name = "futures-channel" version = "0.3.31" @@ -180,6 +221,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10" dependencies = [ "futures-core", + "futures-sink", ] [[package]] @@ -188,6 +230,40 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e" +[[package]] +name = "futures-executor" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f" +dependencies = [ + "futures-core", + "futures-task", + "futures-util", +] + +[[package]] +name = "futures-io" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6" + +[[package]] +name = "futures-macro" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650" +dependencies = [ + "proc-macro2", + "quote", + "syn", +] + +[[package]] +name = "futures-sink" +version = "0.3.31" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7" + [[package]] name = "futures-task" version = "0.3.31" @@ -200,10 +276,16 @@ version = "0.3.31" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81" dependencies = [ + "futures-channel", "futures-core", + "futures-io", + "futures-macro", + "futures-sink", "futures-task", + "memchr", "pin-project-lite", "pin-utils", + "slab", ] [[package]] @@ -252,6 +334,16 @@ dependencies = [ "pin-project-lite", ] +[[package]] +name = "http-serde" +version = "2.1.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0f056c8559e3757392c8d091e796416e4649d8e49e88b8d76df6c002f05027fd" +dependencies = [ + "http", + "serde", +] + [[package]] name = "httparse" version = "1.9.5" @@ -387,6 +479,23 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "multer" +version = "3.1.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "83e87776546dc87511aa5ee218730c92b666d7264ab6ed41f9d215af9cd5224b" +dependencies = [ + "bytes", + "encoding_rs", + "futures-util", + "http", + "httparse", + "memchr", + "mime", + "spin", + "version_check", +] + [[package]] name = "nu-ansi-term" version = "0.46.0" @@ -482,6 +591,19 @@ version = "0.1.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184" +[[package]] +name = "problem_details" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7ee2055bf7d8f34bd11bf85388b878bf244256015f1ed290b6b717c0e97bb478" +dependencies = [ + "axum", + "http", + "http-serde", + "serde", + "serde_json", +] + [[package]] name = "proc-macro2" version = "1.0.92" @@ -627,6 +749,15 @@ dependencies = [ "libc", ] +[[package]] +name = "slab" +version = "0.4.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f92a496fb766b417c996b9c5e57daf2f7ad3b0bebe1ccfca4856390e3d3bb67" +dependencies = [ + "autocfg", +] + [[package]] name = "smallvec" version = "1.13.2" @@ -643,6 +774,12 @@ dependencies = [ "windows-sys", ] +[[package]] +name = "spin" +version = "0.9.8" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "6980e8d7511241f8acf4aebddbb1ff938df5eebe98691418c4468d0b72a96a67" + [[package]] name = "syn" version = "2.0.95" @@ -699,6 +836,17 @@ dependencies = [ "syn", ] +[[package]] +name = "tokio-stream" +version = "0.1.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eca58d7bba4a75707817a2c44174253f9236b2d5fbd055602e9d5c07c139a047" +dependencies = [ + "futures-core", + "pin-project-lite", + "tokio", +] + [[package]] name = "toml" version = "0.8.19" diff --git a/casket-backend/Cargo.toml b/casket-backend/Cargo.toml index 5379bc0..a84789d 100644 --- a/casket-backend/Cargo.toml +++ b/casket-backend/Cargo.toml @@ -2,12 +2,18 @@ name = "casket-backend" version = "0.1.0" edition = "2021" +description = "Backend for the casket file cloud." +repository = "https://git.nifni.eu/nif/casket" [dependencies] -axum = { version = "0.8.1" } +axum = { version = "0.8.1", features = ["multipart", "macros"] } tokio = { version = "1.42.0", features = ["full"] } +tokio-stream = "0.1.17" +futures = "0.3" +futures-util = "0.3" figment = { version = "0.10.19", features = ["toml", "env"] } -serde = {version = "1.0.217", features = ["derive"]} +serde = { version = "1.0.217", features = ["derive"] } tracing = "0.1.41" tracing-log = "0.2.0" -tracing-subscriber = "0.3.19" \ No newline at end of file +tracing-subscriber = "0.3.19" +problem_details = { version = "0.7.0", features = ["axum"] } \ No newline at end of file diff --git a/casket-backend/casket.toml b/casket-backend/casket.toml index e5c18c6..7cec740 100644 --- a/casket-backend/casket.toml +++ b/casket-backend/casket.toml @@ -1,3 +1,6 @@ [server] port = 3000 -bind_address = "127.0.0.1" \ No newline at end of file +bind_address = "127.0.0.1" + +[files] +directory = "/tmp/casket" diff --git a/casket-backend/src/config.rs b/casket-backend/src/config.rs index b8a4f74..d9b8f0c 100644 --- a/casket-backend/src/config.rs +++ b/casket-backend/src/config.rs @@ -1,12 +1,14 @@ use figment::providers::{Env, Format}; use figment::{providers::Toml, Figment}; use serde::Deserialize; +use std::path::PathBuf; pub const CONFIG_LOCATIONS: [&str; 2] = ["casket-backend/casket.toml", "/config/casket.toml"]; #[derive(Deserialize, Clone, Debug)] pub struct Config { pub server: Server, + pub files: Files, } #[derive(Deserialize, Clone, Debug)] @@ -14,6 +16,10 @@ pub struct Server { pub port: u16, pub bind_address: String, } +#[derive(Deserialize, Clone, Debug)] +pub struct Files { + pub directory: PathBuf, +} pub fn get_config() -> figment::Result { CONFIG_LOCATIONS diff --git a/casket-backend/src/errors.rs b/casket-backend/src/errors.rs new file mode 100644 index 0000000..40505eb --- /dev/null +++ b/casket-backend/src/errors.rs @@ -0,0 +1,4 @@ +pub const ERROR_DETAILS_PATH_TRAVERSAL: &str = "You must not traverse!"; +pub const ERROR_DETAILS_INTERNAL_ERROR: &str = + "Internal Error! Check the logs of the backend service for details."; +pub const ERROR_DETAILS_ABSOLUTE_PATH_NOT_ALLOWED: &str = "Absolute paths are not allowed."; diff --git a/casket-backend/src/files/mod.rs b/casket-backend/src/files/mod.rs new file mode 100644 index 0000000..088678c --- /dev/null +++ b/casket-backend/src/files/mod.rs @@ -0,0 +1,181 @@ +use crate::{errors, AppState}; +use axum::body::Bytes; +use axum::debug_handler; +use axum::extract::multipart::{Field, MultipartError}; +use axum::extract::{Multipart, State}; +use axum::http::StatusCode; +use futures_util::stream::MapErr; +use futures_util::{StreamExt, TryStreamExt}; +use problem_details::ProblemDetails; +use std::fmt::Debug; +use std::io; +use std::io::Error; +use std::path::{Component, Path, PathBuf}; +use tokio::fs::{create_dir_all, File}; +use tokio::io::AsyncWriteExt; +use tokio_stream::Stream; +use tracing::error; + +#[debug_handler] +pub async fn handle_file_uploads( + State(state): State, + axum::extract::Path(user_id): axum::extract::Path, + mut multipart: Multipart, +) -> Result<(), ProblemDetails> { + while let Some(field) = multipart.next_field().await.unwrap() { + handle_single_file_upload(&state, &user_id, field).await?; + } + Ok(()) +} + +// clippy behaves weird. When removing the lifetime the compilation fails since the Field type has +// a generic lifetime around the multipart. +#[allow(clippy::needless_lifetimes)] +async fn handle_single_file_upload<'field_lifetime>( + state: &AppState, + user_id: &str, + field: Field<'field_lifetime>, +) -> Result<(), ProblemDetails> { + let path = field.name().unwrap().to_string(); + let filesystem_path = build_system_path(&state.config.files.directory, user_id, &path)?; + save_file(filesystem_path, map_error_to_io_error(field)) + .await + .map_err(to_internal_error) +} + +pub async fn save_file( + path: PathBuf, + mut content: impl Stream> + Unpin, +) -> Result<(), Error> { + create_dir_all(path.parent().unwrap()).await?; + let mut file = File::create(path).await?; + while let Some(bytes) = content.next().await { + file.write_all(&bytes?).await?; + } + Ok(()) +} + +fn to_internal_error(error: impl Debug) -> ProblemDetails { + error!("{:?}", error); + ProblemDetails::from_status_code(StatusCode::INTERNAL_SERVER_ERROR) + .with_detail(errors::ERROR_DETAILS_INTERNAL_ERROR) +} + +fn map_error_to_io_error(field: Field) -> MapErr Error> { + field.map_err(|err| Error::new(io::ErrorKind::Other, err)) +} + +fn build_system_path( + base_directory: &Path, + user_id: &str, + path: &str, +) -> Result { + let user_path = PathBuf::from(path); + if user_path.is_absolute() { + Err( + ProblemDetails::from_status_code(StatusCode::BAD_REQUEST).with_detail( + errors::ERROR_DETAILS_ABSOLUTE_PATH_NOT_ALLOWED.to_owned() + + format!(" Provided Path: {path}").as_str(), + ), + ) + } else { + sanitize_path(&base_directory.join(user_id).join(path)) + } +} + +pub fn sanitize_path(path: &Path) -> Result { + let mut ret = PathBuf::new(); + for component in path.components().peekable() { + match component { + Component::Prefix(..) => unreachable!(), + Component::RootDir => { + ret.push(Component::RootDir); + } + Component::CurDir => {} + Component::ParentDir => { + return Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST) + .with_detail(errors::ERROR_DETAILS_PATH_TRAVERSAL)); + } + Component::Normal(c) => { + ret.push(c); + } + } + } + Ok(ret) +} + +#[cfg(test)] +mod tests { + use super::*; + + #[test] + fn test_sanitize_path_success() { + let cases = &[ + ("", ""), + (".", ""), + (".////./.", ""), + ("/", "/"), + ("/foo/bar", "/foo/bar"), + ("/foo/bar/", "/foo/bar"), + ("/foo/bar/./././///", "/foo/bar"), + ("foo/bar", "foo/bar"), + ("foo/bar/", "foo/bar"), + ("foo/bar/./././///", "foo/bar"), + ]; + for (input, expected) in cases { + let actual = sanitize_path(&PathBuf::from(input)); + assert_eq!(actual, Ok(PathBuf::from(expected)), "input: {input}"); + } + } + + #[test] + fn test_sanitize_path_error() { + let cases = &[ + "..", + "/../", + "../../foo/bar", + "../../foo/bar/", + "../../foo/bar/./././///", + "../../foo/bar/..", + "../../foo/bar/../..", + "../../foo/bar/../../..", + "/foo/bar/../../..", + "/foo/bar/../..", + "/foo/bar/..", + ]; + for input in cases { + let actual = sanitize_path(&PathBuf::from(input)); + assert_eq!( + actual, + Err(ProblemDetails::from_status_code(StatusCode::BAD_REQUEST) + .with_detail(errors::ERROR_DETAILS_PATH_TRAVERSAL)), + "input: {input}" + ); + } + } + + #[test] + fn test_build_system_path_success() { + let input_path = "tmp/blub"; + let user_id = "bla"; + assert_eq!( + build_system_path(&PathBuf::from("/tmp/bla"), user_id, input_path).unwrap(), + PathBuf::from("/tmp/bla/bla/tmp/blub") + ); + } + + #[test] + fn test_build_system_path_error_with_absolute_user_path_returns_error() { + let input_path = "tmp/blub"; + let user_id = "bla"; + assert_eq!( + build_system_path(&PathBuf::from("/tmp/bla"), user_id, input_path), + Err( + ProblemDetails::from_status_code(StatusCode::BAD_REQUEST).with_detail( + errors::ERROR_DETAILS_ABSOLUTE_PATH_NOT_ALLOWED.to_owned() + + format!(" Provided Path: {input_path}").as_str() + ) + ) + ); + } +} diff --git a/casket-backend/src/main.rs b/casket-backend/src/main.rs index ddf1065..eac4644 100644 --- a/casket-backend/src/main.rs +++ b/casket-backend/src/main.rs @@ -1,4 +1,8 @@ +#![warn(clippy::pedantic)] + mod config; +mod errors; +mod files; mod routes; use axum::Router; @@ -8,7 +12,9 @@ use std::str::FromStr; use tracing::{debug, error, info}; #[derive(Clone)] -pub struct AppState {} +pub struct AppState { + config: config::Config, +} #[tokio::main] async fn main() { @@ -22,7 +28,7 @@ async fn main() { let bind = format!("{}:{}", &config.server.bind_address, &config.server.port); let app = Router::new() .merge(routes::routes()) - .with_state(AppState {}); + .with_state(AppState { config }); let listener = tokio::net::TcpListener::bind(bind).await.unwrap(); info!("listening on {}", listener.local_addr().unwrap()); @@ -43,7 +49,7 @@ fn get_log_level() -> tracing::Level { let env = env::var("CASKET_LOG_LEVEL"); match env { Ok(value) => tracing::Level::from_str(&value) - .unwrap_or_else(|_| panic!("Failed to parse log level env {}", value)), + .unwrap_or_else(|_| panic!("Failed to parse log level env {value}")), Err(_) => tracing::Level::INFO, } } diff --git a/casket-backend/src/routes/mod.rs b/casket-backend/src/routes/mod.rs index 0b41156..4f369e5 100644 --- a/casket-backend/src/routes/mod.rs +++ b/casket-backend/src/routes/mod.rs @@ -1,9 +1,12 @@ +use crate::files; +use crate::AppState; +use axum::routing::post; use axum::{response::Html, routing::get, Router}; -use crate::AppState; - pub fn routes() -> Router { - Router::new().route("/", get(handler)) + Router::new() + .route("/", get(handler)) + .route("/api/v1/user/{:user_id}/files", post(files::handle_file_uploads)) } async fn handler() -> Html<&'static str> {