From 3df21e8257b5043b943630307e4110d2f188cc06 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 4 Sep 2024 13:01:23 +0200 Subject: [PATCH 1/4] fix: old media used spaces in content disposition without quotes --- src/api/client_server/media.rs | 14 ++++---- src/database/mod.rs | 61 +++++++++++++++++++++++++++++++++- 2 files changed, 67 insertions(+), 8 deletions(-) diff --git a/src/api/client_server/media.rs b/src/api/client_server/media.rs index ada2bd7c..3f483c2b 100644 --- a/src/api/client_server/media.rs +++ b/src/api/client_server/media.rs @@ -188,11 +188,11 @@ async fn get_content( ) -> Result { let mxc = format!("mxc://{}/{}", server_name, media_id); - if let Some(FileMeta { + if let Ok(Some(FileMeta { content_disposition, content_type, file, - }) = services().media.get(mxc.clone()).await? + })) = services().media.get(mxc.clone()).await { Ok(get_content::v1::Response { file, @@ -264,9 +264,9 @@ async fn get_content_as_filename( ) -> Result { let mxc = format!("mxc://{}/{}", server_name, media_id); - if let Some(FileMeta { + if let Ok(Some(FileMeta { file, content_type, .. - }) = services().media.get(mxc.clone()).await? + })) = services().media.get(mxc.clone()).await { Ok(get_content_as_filename::v1::Response { file, @@ -348,9 +348,9 @@ async fn get_content_thumbnail( ) -> Result { let mxc = format!("mxc://{}/{}", server_name, media_id); - if let Some(FileMeta { + if let Ok(Some(FileMeta { file, content_type, .. - }) = services() + })) = services() .media .get_thumbnail( mxc.clone(), @@ -361,7 +361,7 @@ async fn get_content_thumbnail( .try_into() .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, ) - .await? + .await { Ok(get_content_thumbnail::v1::Response { file, content_type }) } else if server_name != services().globals.server_name() && allow_remote { diff --git a/src/database/mod.rs b/src/database/mod.rs index 5171d4bb..a8d44820 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -6,6 +6,7 @@ use crate::{ SERVICES, }; use abstraction::{KeyValueDatabaseEngine, KvTree}; +use base64::{engine::general_purpose, Engine}; use directories::ProjectDirs; use lru_cache::LruCache; @@ -424,7 +425,7 @@ impl KeyValueDatabase { } // If the database has any data, perform data migrations before starting - let latest_database_version = 13; + let latest_database_version = 14; if services().users.count()? > 0 { // MIGRATIONS @@ -941,6 +942,64 @@ impl KeyValueDatabase { warn!("Migration: 12 -> 13 finished"); } + if services().globals.database_version()? < 14 { + // Reconstruct all media using the filesystem + db.mediaid_file.clear().unwrap(); + + for file in fs::read_dir(services().globals.get_media_folder()).unwrap() { + let file = file.unwrap(); + let mediaid = general_purpose::URL_SAFE_NO_PAD + .decode(file.file_name().into_string().unwrap()) + .unwrap(); + + let mut parts = mediaid.rsplit(|&b| b == 0xff); + + let mut removed_bytes = 0; + + let content_type_bytes = parts.next().unwrap(); + removed_bytes += content_type_bytes.len() + 1; + + let content_disposition_bytes = parts + .next() + .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?; + removed_bytes += content_disposition_bytes.len(); + + let mut content_disposition = + utils::string_from_bytes(content_disposition_bytes).map_err(|_| { + Error::bad_database("Content Disposition in mediaid_file is invalid.") + })?; + + if content_disposition.contains("filename=") + && !content_disposition.contains("filename=\"") + { + println!("{}", &content_disposition); + content_disposition = + content_disposition.replacen("filename=", "filename=\"", 1); + content_disposition.push('"'); + println!("{}", &content_disposition); + + let mut new_key = mediaid[..(mediaid.len() - removed_bytes)].to_vec(); + assert!(*new_key.last().unwrap() == 0xff); + + new_key.extend_from_slice(content_disposition.to_string().as_bytes()); + new_key.push(0xff); + new_key.extend_from_slice(content_type_bytes); + + // Some file names are too long. Ignore those. + let _ = fs::rename( + services().globals.get_media_file(&mediaid), + services().globals.get_media_file(&new_key), + ); + db.mediaid_file.insert(&new_key, &[])?; + } else { + db.mediaid_file.insert(&mediaid, &[])?; + } + } + services().globals.bump_database_version(14)?; + + warn!("Migration: 13 -> 14 finished"); + } + assert_eq!( services().globals.database_version().unwrap(), latest_database_version From a7405cddc020817614f1186903c0fc95c2403dce Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 24 Sep 2024 18:48:48 +0200 Subject: [PATCH 2/4] fix: Matrix media repo --- Cargo.lock | 27 ++++++++++---------- src/api/client_server/membership.rs | 2 +- src/api/server_server.rs | 13 +++++----- src/database/mod.rs | 38 ++++++++++++++++++++++------- 4 files changed, 50 insertions(+), 30 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index 20082187..19e75ceb 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -2232,7 +2232,7 @@ dependencies = [ [[package]] name = "ruma" version = "0.10.1" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "assign", "js_int", @@ -2253,7 +2253,7 @@ dependencies = [ [[package]] name = "ruma-appservice-api" version = "0.10.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "js_int", "ruma-common", @@ -2265,7 +2265,7 @@ dependencies = [ [[package]] name = "ruma-client-api" version = "0.18.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "as_variant", "assign", @@ -2288,7 +2288,7 @@ dependencies = [ [[package]] name = "ruma-common" version = "0.13.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "as_variant", "base64 0.22.1", @@ -2318,7 +2318,7 @@ dependencies = [ [[package]] name = "ruma-events" version = "0.28.1" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "as_variant", "indexmap 2.2.6", @@ -2341,7 +2341,7 @@ dependencies = [ [[package]] name = "ruma-federation-api" version = "0.9.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "bytes", "http 1.1.0", @@ -2359,7 +2359,7 @@ dependencies = [ [[package]] name = "ruma-identifiers-validation" version = "0.9.5" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "js_int", "thiserror", @@ -2368,7 +2368,7 @@ dependencies = [ [[package]] name = "ruma-identity-service-api" version = "0.9.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "js_int", "ruma-common", @@ -2378,8 +2378,9 @@ dependencies = [ [[package]] name = "ruma-macros" version = "0.13.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ + "cfg-if", "once_cell", "proc-macro-crate", "proc-macro2", @@ -2393,7 +2394,7 @@ dependencies = [ [[package]] name = "ruma-push-gateway-api" version = "0.9.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "js_int", "ruma-common", @@ -2405,7 +2406,7 @@ dependencies = [ [[package]] name = "ruma-server-util" version = "0.3.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "headers", "http 1.1.0", @@ -2418,7 +2419,7 @@ dependencies = [ [[package]] name = "ruma-signatures" version = "0.15.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "base64 0.22.1", "ed25519-dalek", @@ -2434,7 +2435,7 @@ dependencies = [ [[package]] name = "ruma-state-res" version = "0.11.0" -source = "git+https://github.com/ruma/ruma#82417e394076440089cd8ada87485d9a44cc4ba0" +source = "git+https://github.com/ruma/ruma#c06af4385e0e30c48a8e9ca3d488da32102d0db9" dependencies = [ "itertools", "js_int", diff --git a/src/api/client_server/membership.rs b/src/api/client_server/membership.rs index 3f3d25d5..baf2f239 100644 --- a/src/api/client_server/membership.rs +++ b/src/api/client_server/membership.rs @@ -97,7 +97,7 @@ pub async fn join_room_by_id_or_alias_route( let (servers, room_id) = match OwnedRoomId::try_from(body.room_id_or_alias) { Ok(room_id) => { - let mut servers = body.server_name.clone(); + let mut servers = body.via.clone(); servers.extend( services() .rooms diff --git a/src/api/server_server.rs b/src/api/server_server.rs index 56dd74d3..f6dc58fe 100644 --- a/src/api/server_server.rs +++ b/src/api/server_server.rs @@ -10,7 +10,7 @@ use crate::{ services, utils, Error, PduEvent, Result, Ruma, }; use axum::{response::IntoResponse, Json}; -use axum_extra::headers::{authorization::Credentials, CacheControl, Header}; +use axum_extra::headers::{CacheControl, Header}; use get_profile_information::v1::ProfileField; use http::header::AUTHORIZATION; @@ -52,7 +52,6 @@ use ruma::{ StateEventType, TimelineEventType, }, serde::{Base64, JsonObject, Raw}, - server_util::authorization::XMatrix, to_device::DeviceIdOrAllDevices, uint, user_id, CanonicalJsonObject, CanonicalJsonValue, EventId, MilliSecondsSinceUnixEpoch, OwnedEventId, OwnedRoomId, OwnedServerName, OwnedServerSigningKeyId, OwnedUserId, RoomId, @@ -275,15 +274,15 @@ where for s in signature_server { http_request.headers_mut().insert( AUTHORIZATION, - XMatrix::parse(&format!( + format!( "X-Matrix origin=\"{}\",destination=\"{}\",key=\"{}\",sig=\"{}\"", services().globals.server_name(), destination, s.0, s.1 - )) - .expect("When Ruma signs JSON, it produces a valid base64 signature. All other types are valid ServerNames or OwnedKeyId") - .encode(), + ) + .try_into() + .unwrap(), ); } } @@ -343,7 +342,7 @@ where response.map_err(|e| { warn!( - "Invalid 200 response from {} on: {} {}", + "Invalid 200 response from {} on: {} {:?}", &destination, url, e ); Error::BadServerResponse("Server returned bad 200 response.") diff --git a/src/database/mod.rs b/src/database/mod.rs index a8d44820..1caf61ca 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -425,7 +425,7 @@ impl KeyValueDatabase { } // If the database has any data, perform data migrations before starting - let latest_database_version = 14; + let latest_database_version = 15; if services().users.count()? > 0 { // MIGRATIONS @@ -942,7 +942,7 @@ impl KeyValueDatabase { warn!("Migration: 12 -> 13 finished"); } - if services().globals.database_version()? < 14 { + if services().globals.database_version()? < 15 { // Reconstruct all media using the filesystem db.mediaid_file.clear().unwrap(); @@ -981,23 +981,43 @@ impl KeyValueDatabase { let mut new_key = mediaid[..(mediaid.len() - removed_bytes)].to_vec(); assert!(*new_key.last().unwrap() == 0xff); + let mut shorter_key = new_key.clone(); + shorter_key.extend( + ruma::http_headers::ContentDisposition::new( + ruma::http_headers::ContentDispositionType::Inline, + ) + .to_string() + .as_bytes(), + ); + shorter_key.push(0xff); + shorter_key.extend_from_slice(content_type_bytes); + new_key.extend_from_slice(content_disposition.to_string().as_bytes()); new_key.push(0xff); new_key.extend_from_slice(content_type_bytes); // Some file names are too long. Ignore those. - let _ = fs::rename( + match fs::rename( services().globals.get_media_file(&mediaid), services().globals.get_media_file(&new_key), - ); - db.mediaid_file.insert(&new_key, &[])?; - } else { - db.mediaid_file.insert(&mediaid, &[])?; + ) { + Ok(_) => { + db.mediaid_file.insert(&mediaid, &[])?; + } + Err(_) => { + fs::rename( + services().globals.get_media_file(&mediaid), + services().globals.get_media_file(&shorter_key), + ) + .unwrap(); + db.mediaid_file.insert(&shorter_key, &[])?; + } + } } } - services().globals.bump_database_version(14)?; + services().globals.bump_database_version(15)?; - warn!("Migration: 13 -> 14 finished"); + warn!("Migration: 13 -> 15 finished"); } assert_eq!( From fea85b0894d616ae963b1a8f4319040001fa8005 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Tue, 24 Sep 2024 23:00:37 +0200 Subject: [PATCH 3/4] fix: Migration typo for media --- src/api/client_server/media.rs | 2 +- src/database/mod.rs | 12 +++++++----- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/src/api/client_server/media.rs b/src/api/client_server/media.rs index 3f483c2b..03e4cbab 100644 --- a/src/api/client_server/media.rs +++ b/src/api/client_server/media.rs @@ -359,7 +359,7 @@ async fn get_content_thumbnail( .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, height .try_into() - .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Width is invalid."))?, + .map_err(|_| Error::BadRequest(ErrorKind::InvalidParam, "Height is invalid."))?, ) .await { diff --git a/src/database/mod.rs b/src/database/mod.rs index 1caf61ca..0035862a 100644 --- a/src/database/mod.rs +++ b/src/database/mod.rs @@ -425,7 +425,7 @@ impl KeyValueDatabase { } // If the database has any data, perform data migrations before starting - let latest_database_version = 15; + let latest_database_version = 16; if services().users.count()? > 0 { // MIGRATIONS @@ -942,7 +942,7 @@ impl KeyValueDatabase { warn!("Migration: 12 -> 13 finished"); } - if services().globals.database_version()? < 15 { + if services().globals.database_version()? < 16 { // Reconstruct all media using the filesystem db.mediaid_file.clear().unwrap(); @@ -1002,7 +1002,7 @@ impl KeyValueDatabase { services().globals.get_media_file(&new_key), ) { Ok(_) => { - db.mediaid_file.insert(&mediaid, &[])?; + db.mediaid_file.insert(&new_key, &[])?; } Err(_) => { fs::rename( @@ -1013,11 +1013,13 @@ impl KeyValueDatabase { db.mediaid_file.insert(&shorter_key, &[])?; } } + } else { + db.mediaid_file.insert(&mediaid, &[])?; } } - services().globals.bump_database_version(15)?; + services().globals.bump_database_version(16)?; - warn!("Migration: 13 -> 15 finished"); + warn!("Migration: 13 -> 16 finished"); } assert_eq!( From 65fe6b0ab5968e4b6c4fdb1a20070ddd4def08ca Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Timo=20K=C3=B6sters?= Date: Wed, 25 Sep 2024 09:06:43 +0200 Subject: [PATCH 4/4] fix: Empty content dispositions could create problems --- src/database/key_value/media.rs | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/src/database/key_value/media.rs b/src/database/key_value/media.rs index 52a8e79e..99df0097 100644 --- a/src/database/key_value/media.rs +++ b/src/database/key_value/media.rs @@ -63,9 +63,9 @@ impl service::media::Data for KeyValueDatabase { .next() .ok_or_else(|| Error::bad_database("Media ID in db is invalid."))?; - let content_disposition = content_disposition_bytes - .try_into() - .map_err(|_| Error::bad_database("Content Disposition in mediaid_file is invalid."))?; + let content_disposition = content_disposition_bytes.try_into().unwrap_or_else(|_| { + ContentDisposition::new(ruma::http_headers::ContentDispositionType::Inline) + }); Ok((content_disposition, content_type, key)) } }