From a64072bf2a2a169a03fce44b35a0a697db6c429d Mon Sep 17 00:00:00 2001
From: Pierre Jarriges <pierre.jarriges@tutanota.com>
Date: Wed, 20 Jul 2022 15:57:14 +0200
Subject: [PATCH] wip static files indexation
---
.gitignore | 2 +-
src/core.rs | 1 +
src/core/static_files.rs | 212 ++++++++++++++++++++++++++++++++++++
src/main.rs | 7 ++
src/service/static_files.rs | 120 +++++++++++---------
5 files changed, 292 insertions(+), 50 deletions(-)
create mode 100644 src/core.rs
create mode 100644 src/core/static_files.rs
diff --git a/.gitignore b/.gitignore
index 7bbcb76..79d0212 100644
--- a/.gitignore
+++ b/.gitignore
@@ -9,4 +9,4 @@ public/**/*.js
public/**/view/*
public/standard/test_sitemap.xml
public/standard/dyn_sitemap.xml
-public/uploads
\ No newline at end of file
+public/assets/uploads
\ No newline at end of file
diff --git a/src/core.rs b/src/core.rs
new file mode 100644
index 0000000..13f71bd
--- /dev/null
+++ b/src/core.rs
@@ -0,0 +1 @@
+pub mod static_files;
diff --git a/src/core/static_files.rs b/src/core/static_files.rs
new file mode 100644
index 0000000..84ac310
--- /dev/null
+++ b/src/core/static_files.rs
@@ -0,0 +1,212 @@
+use crate::env::Env;
+use crate::AppState;
+use std::fs::create_dir_all;
+use std::path::Path;
+
+#[derive(Debug, Clone)]
+pub struct StaticFilesIndex(pub Vec<String>);
+
+impl StaticFilesIndex {
+ fn rec_read_dir(root: &Path, files: &mut Vec<String>, strip_from: &Path) {
+ for entry in root.read_dir().unwrap() {
+ if let Ok(entry) = entry {
+ if entry.path().is_dir() {
+ StaticFilesIndex::rec_read_dir(&entry.path(), files, strip_from);
+ } else {
+ StaticFilesIndex::_push_path(&entry.path(), files, strip_from);
+ }
+ }
+ }
+ }
+
+ fn _push_path(path: &Path, files: &mut Vec<String>, strip_from: &Path) {
+ let push_path = path.strip_prefix(strip_from).unwrap();
+ files.push(push_path.to_str().unwrap().to_owned());
+ }
+
+ pub fn rebuild(&mut self, env: &Env) {
+ let root = env.public_dir.join("assets");
+ self.0 = Vec::new();
+ StaticFilesIndex::rec_read_dir(&root, &mut self.0, &env.public_dir);
+ }
+
+ pub fn push_path(&mut self, path: &Path, env: &Env) {
+ let strip_from = env.public_dir.join("assets");
+ StaticFilesIndex::_push_path(path, &mut self.0, &strip_from);
+ }
+}
+
+#[derive(Debug, PartialEq)]
+pub enum UploadType {
+ Image,
+ Sound,
+ Video,
+ Doc,
+}
+
+pub struct UploadData {
+ pub up_type: UploadType,
+ pub filename: String,
+}
+
+pub fn create_dir_if_missing(path: std::path::PathBuf) -> std::path::PathBuf {
+ if !path.exists() {
+ create_dir_all(&path).unwrap();
+ }
+
+ path
+}
+
+pub fn get_uploads_dir(app_state: &AppState) -> std::path::PathBuf {
+ create_dir_if_missing(app_state.env.public_dir.join("assets/uploads"))
+}
+
+pub fn dirname_from_type(upload_type: &UploadType) -> String {
+ match upload_type {
+ UploadType::Image => String::from("images"),
+ UploadType::Sound => String::from("sounds"),
+ UploadType::Video => String::from("videos"),
+ UploadType::Doc => String::from("docs"),
+ }
+}
+
+pub fn upload_type_from_file_ext(ext: &String) -> UploadType {
+ match &ext[..] {
+ "webp" | "jpg" | "png" | "jpeg" | "bmp" | "gif" => UploadType::Image,
+ "mp3" | "ogg" | "wav" | "opus" => UploadType::Sound,
+ "mp4" | "webm" | "ogv" => UploadType::Video,
+ _ => UploadType::Doc,
+ }
+}
+
+pub fn file_ext(file_name: &String) -> Result<String, String> {
+ let parts = file_name.split(".").collect::<Vec<&str>>();
+
+ let err_msg = format!("Couldn't get extension from filename : {}", file_name);
+
+ if parts.len() < 2 {
+ return Err(err_msg);
+ }
+
+ match parts.last() {
+ Some(ext) => Ok(ext.to_string()),
+ None => Err(err_msg),
+ }
+}
+
+/*@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*@@
+ *@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*@@
+ * _______ ______ ______ _______ *@@
+ * |__ __@ | ____@ / ____@ |__ __@ *@@
+ * | @ | @__ \_ @_ | @ *@@
+ * | @ | __@ \ @_ | @ *@@
+ * | @ | @___ ____\ @ | @ *@@
+ * |__@ |______@ \______@ |__@ *@@
+ * *@@
+ *@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*@@
+ *@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@*@*/
+
+#[cfg(test)]
+mod test_static_files {
+ use super::*;
+ use std::{fs::remove_dir_all, path::PathBuf};
+
+ #[test]
+ fn should_create_missing_directory() {
+ let missing_path = PathBuf::from("./some_nested/missing_dir");
+ let created_path = create_dir_if_missing(missing_path);
+ assert!(created_path.exists());
+ remove_dir_all(PathBuf::from("./some_nested")).unwrap();
+ }
+
+ #[test]
+ fn uploads_subdirs_should_be_consistent() {
+ assert_eq!(
+ upload_type_from_file_ext(&"jpg".to_string()),
+ UploadType::Image
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"jpeg".to_string()),
+ UploadType::Image
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"png".to_string()),
+ UploadType::Image
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"bmp".to_string()),
+ UploadType::Image
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"gif".to_string()),
+ UploadType::Image
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"webp".to_string()),
+ UploadType::Image
+ );
+
+ assert_eq!(
+ upload_type_from_file_ext(&"mp3".to_string()),
+ UploadType::Sound
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"ogg".to_string()),
+ UploadType::Sound
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"wav".to_string()),
+ UploadType::Sound
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"opus".to_string()),
+ UploadType::Sound
+ );
+
+ assert_eq!(
+ upload_type_from_file_ext(&"mp4".to_string()),
+ UploadType::Video
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"webm".to_string()),
+ UploadType::Video
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"ogv".to_string()),
+ UploadType::Video
+ );
+
+ assert_eq!(
+ upload_type_from_file_ext(&"any".to_string()),
+ UploadType::Doc
+ );
+ assert_eq!(
+ upload_type_from_file_ext(&"jszaj".to_string()),
+ UploadType::Doc
+ );
+ }
+
+ #[test]
+ fn should_get_filename_extension() {
+ fn valid_ext(ext: Result<String, String>) -> String {
+ assert!(ext.is_ok());
+ ext.unwrap()
+ }
+
+ let filename = String::from("somefilename.png");
+ let ext = valid_ext(file_ext(&filename));
+ assert_eq!(ext, "png");
+
+ let filename = String::from("aïe aïe aïe.wav");
+ let ext = valid_ext(file_ext(&filename));
+ assert_eq!(ext, "wav");
+
+ let filename = String::from("salut Ça va.machin.jpg");
+ let ext = valid_ext(file_ext(&filename));
+ assert_eq!(ext, "jpg");
+
+ let filename = String::from("no extension");
+ let ext = file_ext(&filename);
+ assert!(ext.is_err());
+ }
+}
diff --git a/src/main.rs b/src/main.rs
index 99b3a57..6439d51 100644
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,5 +1,6 @@
//! # WEB SERVER FOR THE KUADRADO SOFTWARE WEBSITE
mod app_state;
+mod core;
mod crypto;
mod env;
mod init_admin;
@@ -11,6 +12,7 @@ mod static_view;
mod tls;
mod view;
mod view_resource;
+use crate::core::static_files::StaticFilesIndex;
use actix_files::Files;
use actix_web::{
middleware::{normalize::TrailingSlash, Logger, NormalizePath},
@@ -37,6 +39,10 @@ async fn main() -> std::io::Result<()> {
let server_port_tls = env_var("SERVER_PORT_TLS").expect("SERVER_PORT_TLS is not defined.");
let app_state = AppState::with_default_admin_user().await;
+ let mut static_files_index = StaticFilesIndex(Vec::new());
+ static_files_index.rebuild(&app_state.env);
+ let static_files_index = Data::new(std::sync::Mutex::new(static_files_index));
+
HttpServer::new(move || {
App::new()
.wrap(Logger::default())
@@ -47,6 +53,7 @@ async fn main() -> std::io::Result<()> {
)]))
.wrap(actix_web::middleware::Compress::default())
.app_data(Data::new(app_state.clone()))
+ .app_data(Data::clone(&static_files_index))
.app_data(Data::new(AuthenticatedAdminMiddleware::new(
"kuadrado-admin-auth",
)))
diff --git a/src/service/static_files.rs b/src/service/static_files.rs
index a179b2b..b9c133b 100644
--- a/src/service/static_files.rs
+++ b/src/service/static_files.rs
@@ -1,16 +1,65 @@
-use crate::{middleware::AuthenticatedAdminMiddleware, AppState};
+use crate::{core::static_files::*, middleware::AuthenticatedAdminMiddleware, AppState};
use actix_multipart::Multipart;
use actix_web::{post, web::Data, HttpRequest, HttpResponse, Responder};
use futures::StreamExt;
use std::{
- fs::{create_dir_all, remove_file, File},
+ fs::{remove_file, File},
io::Write,
};
-// TODO separate file writing logic from Multipart implementation and implement the unit tests
+fn upload_data_from_multipart_field(field: &actix_multipart::Field) -> Result<UploadData, String> {
+ match field.content_disposition() {
+ Some(content_disposition) => match content_disposition.get_filename() {
+ Some(fname) => match file_ext(&fname.to_string()) {
+ Ok(ext) => Ok(UploadData {
+ up_type: upload_type_from_file_ext(&ext),
+ filename: fname.to_owned(),
+ }),
+ Err(msg) => return Err(msg),
+ },
+ None => Err("Couldn't retrieve file extension".to_string()),
+ },
+ None => Err("Missing content disposition".to_string()),
+ }
+}
+
+async fn write_uploaded_file(
+ app_state: &AppState,
+ field: &mut actix_multipart::Field,
+ filename: &String,
+ upload_type: UploadType,
+) -> Result<String, String> {
+ let uploads_dir = get_uploads_dir(app_state);
+ let sub_dir = dirname_from_type(&upload_type);
+ let filepath = create_dir_if_missing(uploads_dir.join(&sub_dir)).join(&filename);
+
+ match File::create(&filepath) {
+ Err(e) => Err(format!("Error creating file {:?}", e)),
+ Ok(mut f) => {
+ // Field in turn is stream of *Bytes* object
+ while let Some(chunk) = field.next().await {
+ match chunk {
+ Ok(chunk) => {
+ if f.write_all(&chunk).is_err() {
+ remove_file(&filepath).unwrap();
+ return Err("Error writing chunk".to_string());
+ }
+ }
+ Err(e) => {
+ return Err(format!("Error writing file {} : {:?}", filename, e));
+ }
+ }
+ }
+
+ Ok(filepath.into_os_string().into_string().unwrap())
+ }
+ }
+}
+
#[post("/post-files")]
pub async fn post_files(
app_state: Data<AppState>,
+ static_files_index: Data<std::sync::Mutex<StaticFilesIndex>>,
mut payload: Multipart,
middleware: Data<AuthenticatedAdminMiddleware<'_>>,
req: HttpRequest,
@@ -19,61 +68,34 @@ pub async fn post_files(
return HttpResponse::Unauthorized().finish();
}
- let uploads_dir = app_state.env.public_dir.join("uploads");
-
- if !uploads_dir.exists() {
- create_dir_all(&uploads_dir).unwrap();
- }
-
let mut uploaded_filepathes = Vec::new();
+ let mut files_index = static_files_index.lock().unwrap();
while let Some(item) = payload.next().await {
match item {
Ok(mut field) => {
- // Field in turn is stream of *Bytes* object
- // A multipart/form-data stream has to contain `content_disposition`
- let content_disposition = field
- .content_disposition()
- .expect("Missing Content Disposition header");
-
- let filename = content_disposition
- .get_filename()
- .expect("Missing filename");
-
- let filepath = uploads_dir.join(&filename);
+ let up_data = upload_data_from_multipart_field(&field);
- let f = File::create(&filepath);
-
- if f.is_err() {
- return HttpResponse::InternalServerError()
- .body(format!("Error creating file {:?}", f));
+ if let Err(msg) = up_data {
+ return HttpResponse::InternalServerError().body(msg);
}
- let mut f = f.unwrap();
- let mut error = None;
-
- // Field in turn is stream of *Bytes* object
- 'chunks: while let Some(chunk) = field.next().await {
- match chunk {
- Ok(chunk) => {
- if f.write_all(&chunk).is_err() {
- error = Some("Error writing chunk".to_string());
- break 'chunks;
- }
- }
- Err(e) => {
- error = Some(format!("Error writing file {} : {:?}", filename, e));
- break 'chunks;
- }
+ let up_data = up_data.unwrap();
+
+ match write_uploaded_file(
+ &app_state,
+ &mut field,
+ &up_data.filename,
+ up_data.up_type,
+ )
+ .await
+ {
+ Err(msg) => return HttpResponse::InternalServerError().body(msg),
+ Ok(filepath) => {
+ files_index.push_path(std::path::Path::new(&filepath), &app_state.env);
+ uploaded_filepathes.push(filepath);
}
}
-
- if let Some(err) = error {
- remove_file(&filepath).unwrap();
- return HttpResponse::InternalServerError().body(err);
- }
-
- uploaded_filepathes.push(String::from(filepath.to_string_lossy()));
}
Err(e) => {
return HttpResponse::InternalServerError().body(format!("FIELD ERR {:?}", e))
@@ -84,7 +106,7 @@ pub async fn post_files(
HttpResponse::Ok().json(uploaded_filepathes)
}
-// EXAMPLE FROM ACTIX REPO
+// EXAMPLE FROM ACTIX REPO (using threadpool)
// use futures::TryStreamExt;
// use std::io::Write;
--
GitLab