Merge pull request #57 from bgpkit/features/custom-headers-utils

Custom headers utils

Author: digizeph

CHANGELOG.md

@@ -2,6 +2,22 @@
 
 All notable changes to this project will be documented in this file.
 
+## v0.18.0 -- 2025-05-30
+
+### Highlights
+
+* split `remote` features into `http` and `ftp`, allowing users who only need HTTP or FTP support to use the
+  corresponding feature flag
+    * in most cases, users will likely not need to use the `ftp` feature
+* add `create_client_with_headers` function to allow creating a `reqwest::blocking::Client` with custom headers
+    * this simplifies the process of creating a client with custom headers for HTTP requests
+    * this also allows users to create custom clients without importing `reqwest` crate directly
+* add `rustls_sys` dependency to support `rustls` as the default TLS backend
+
+### Documentation improvements
+
+* update examples on custom HTTP request function `oneio::get_http_reader` to use `create_client_with_headers`
+
 ## v0.17.0 -- 2024-08-04
 
 ### Highlights

Cargo.toml

@@ -1,6 +1,6 @@
 [package]
 name = "oneio"
-version = "0.17.0"
+version = "0.18.0"
 authors = ["Mingwei Zhang <[email protected]>"]
 edition = "2021"
 readme = "README.md"
@@ -43,6 +43,7 @@ zstd = { version = "0.13.2", optional = true }
 # feature: digest
 ring = { version = "0.17", optional = true }
 hex = { version = "0.4", optional = true }
+rustls_sys = { package = "rustls", version = "0.23", optional = true }
 
 # feature: json
 serde = { version = "1.0", optional = true }
@@ -64,6 +65,12 @@ default = ["lib-core", "rustls"]
 
 # library core dependency to enable reading from local/remote with compressions enabled
 lib-core = ["remote", "compressions", "json"]
+
+# remote IO features
+remote = ["http", "ftp"]
+http = ["reqwest"]
+ftp = ["suppaftp"]
+
 # cli dependencies
 cli = [
     # core dependency
@@ -82,7 +89,12 @@ native-tls = [
     "suppaftp?/native-tls",
     "rust-s3?/sync-native-tls",
 ]
-rustls = ["reqwest?/rustls-tls", "suppaftp?/rustls", "rust-s3?/sync-rustls-tls"]
+rustls = [
+    "reqwest?/rustls-tls",
+    "suppaftp?/rustls",
+    "rust-s3?/sync-rustls-tls",
+    "rustls_sys"
+]
 
 digest = ["ring", "hex"]
 
@@ -94,7 +106,6 @@ lz = ["lz4"]
 xz = ["xz2"]
 zstd = ["dep:zstd"]
 
-remote = ["reqwest", "suppaftp"]
 json = ["serde", "serde_json"]
 
 # s3 support, off by default

README.md

@@ -1,24 +1,25 @@
-# OneIO - all-in-one convenient IO library for Rust
+# OneIO - all-in-one IO library for Rust
 
 [![Rust](https://github.com/bgpkit/oneio/actions/workflows/rust.yml/badge.svg)](https://github.com/bgpkit/oneio/actions/workflows/rust.yml)
 [![Crates.io](https://img.shields.io/crates/v/oneio)](https://crates.io/crates/oneio)
 [![Docs.rs](https://docs.rs/oneio/badge.svg)](https://docs.rs/oneio)
 [![License](https://img.shields.io/crates/l/oneio)](https://raw.githubusercontent.com/bgpkit/oneio/main/LICENSE)
 
-OneIO is a Rust library that provides a unified simple IO interface for reading and writing to and from data files from different sources and compressions.
+OneIO is a Rust library that provides a unified IO interface for synchronously reading and writing
+to and from data files from different sources and compressions.
 
 ### Usage and Feature Flags
 
 Enable all compression algorithms and handle remote files (default)
 
 ```toml
-oneio = "0.17"
+oneio = "0.18"
 ```
 
 Select from supported feature flags
 
 ```toml
-oneio = { version = "0.17", default-features = false, features = ["remote", "gz"] }
+oneio = { version = "0.18", default-features = false, features = ["remote", "gz"] }
 ```
 
 Default flags include `lib-core` and `rustls`.
@@ -28,6 +29,8 @@ Default flags include `lib-core` and `rustls`.
 `lib-core` core features include:
 
 - `remote`: allow reading from remote files, including http(s) and ftp
+    - `http`: support reading from http(s) remote files using `reqwest` crate
+    - `ftp`: support reading from ftp remote files using `suppaftp` crate
 - `compressions`: support all compression algorithms
     - `gz`: support `gzip` files using `flate2` crate
     - `bz`: support `bzip2` files using `bzip2` crate
@@ -57,7 +60,7 @@ Users can also manually opt-in to specific compression algorithms. For example,
 and `bzip2` files:
 
 ```toml
-oneio = { version = "0.17", default-features = false, features = ["gz", "bz"] }
+oneio = { version = "0.18", default-features = false, features = ["gz", "bz"] }
 ```
 
 ### Use `oneio` commandline tool
@@ -165,7 +168,7 @@ assert_eq!(lines[1].as_str(), "This is a test.");
 
 ### Use OneIO Writer as a Library
 
-[get_writer] returns a generic writer that implements [Write], and handles decompression from the following types:
+[get_writer] returns a generic writer that implements [std::io::Write], and handles decompression from the following types:
 
 - `gzip`: files ending with `gz` or `gzip`
 - `bzip2`: files ending with `bz` or `bz2`
@@ -204,16 +207,10 @@ std::fs::remove_file(to_write_file).unwrap();
 use std::collections::HashMap;
 use reqwest::header::HeaderMap;
 
-let headers: HeaderMap = (&HashMap::from([("X-Custom-Auth-Key".to_string(), "TOKEN".to_string())]))
-    .try_into().expect("invalid headers");
-
-let client = reqwest::blocking::Client::builder()
-    .default_headers(headers)
-    .danger_accept_invalid_certs(true)
-    .build().unwrap();
+let client = oneio::create_client_with_headers([("X-Custom-Auth-Key", "TOKEN")]).unwrap();
 let mut reader = oneio::get_http_reader(
-    "https://SOME_REMOTE_RESOURCE_PROTECTED_BY_ACCESS_TOKEN",
-    Some(client),
+  "https://SOME_REMOTE_RESOURCE_PROTECTED_BY_ACCESS_TOKEN",
+  Some(client),
 ).unwrap();
 let mut text = "".to_string();
 reader.read_to_string(&mut text).unwrap();

README.tpl

@@ -1,4 +1,4 @@
-# OneIO - all-in-one convenient IO library for Rust
+# OneIO - all-in-one IO library for Rust
 
 [![Rust](https://github.com/bgpkit/oneio/actions/workflows/rust.yml/badge.svg)](https://github.com/bgpkit/oneio/actions/workflows/rust.yml)
 [![Crates.io](https://img.shields.io/crates/v/oneio)](https://crates.io/crates/oneio)

src/bin/oneio.rs

@@ -164,7 +164,7 @@ fn main() {
 
     if cli.download {
         let out_path = match outfile {
-            None => path.split('/').last().unwrap().to_string(),
+            None => path.split('/').next_back().unwrap().to_string(),
             Some(p) => p.to_str().unwrap().to_string(),
         };
 

src/lib.rs

@@ -1,18 +1,19 @@
 /*!
-OneIO is a Rust library that provides a unified simple IO interface for reading and writing to and from data files from different sources and compressions.
+OneIO is a Rust library that provides a unified IO interface for synchronously reading and writing
+to and from data files from different sources and compressions.
 
 ## Usage and Feature Flags
 
 Enable all compression algorithms and handle remote files (default)
 
 ```toml
-oneio = "0.17"
+oneio = "0.18"
 ```
 
 Select from supported feature flags
 
 ```toml
-oneio = { version = "0.17", default-features = false, features = ["remote", "gz"] }
+oneio = { version = "0.18", default-features = false, features = ["remote", "gz"] }
 ```
 
 Default flags include `lib-core` and `rustls`.
@@ -22,6 +23,8 @@ Default flags include `lib-core` and `rustls`.
 `lib-core` core features include:
 
 - `remote`: allow reading from remote files, including http(s) and ftp
+    - `http`: support reading from http(s) remote files using `reqwest` crate
+    - `ftp`: support reading from ftp remote files using `suppaftp` crate
 - `compressions`: support all compression algorithms
     - `gz`: support `gzip` files using `flate2` crate
     - `bz`: support `bzip2` files using `bzip2` crate
@@ -51,7 +54,7 @@ Users can also manually opt-in to specific compression algorithms. For example,
 and `bzip2` files:
 
 ```toml
-oneio = { version = "0.17", default-features = false, features = ["gz", "bz"] }
+oneio = { version = "0.18", default-features = false, features = ["gz", "bz"] }
 ```
 
 ## Use `oneio` commandline tool
@@ -159,7 +162,7 @@ assert_eq!(lines[1].as_str(), "This is a test.");
 
 ## Use OneIO Writer as a Library
 
-[get_writer] returns a generic writer that implements [Write], and handles decompression from the following types:
+[get_writer] returns a generic writer that implements [std::io::Write], and handles decompression from the following types:
 
 - `gzip`: files ending with `gz` or `gzip`
 - `bzip2`: files ending with `bz` or `bz2`
@@ -198,16 +201,10 @@ std::fs::remove_file(to_write_file).unwrap();
 use std::collections::HashMap;
 use reqwest::header::HeaderMap;
 
-let headers: HeaderMap = (&HashMap::from([("X-Custom-Auth-Key".to_string(), "TOKEN".to_string())]))
-    .try_into().expect("invalid headers");
-
-let client = reqwest::blocking::Client::builder()
-    .default_headers(headers)
-    .danger_accept_invalid_certs(true)
-    .build().unwrap();
+let client = oneio::create_client_with_headers([("X-Custom-Auth-Key", "TOKEN")]).unwrap();
 let mut reader = oneio::get_http_reader(
-    "https://SOME_REMOTE_RESOURCE_PROTECTED_BY_ACCESS_TOKEN",
-    Some(client),
+  "https://SOME_REMOTE_RESOURCE_PROTECTED_BY_ACCESS_TOKEN",
+  Some(client),
 ).unwrap();
 let mut text = "".to_string();
 reader.read_to_string(&mut text).unwrap();

src/oneio/mod.rs

@@ -109,7 +109,7 @@ pub fn get_cache_reader(
         path.split('/')
             .collect::<Vec<&str>>()
             .into_iter()
-            .last()
+            .next_back()
             .unwrap()
             .to_string()
     });

src/oneio/remote.rs

@@ -1,3 +1,6 @@
+//! This module provides functionality to handle remote file operations such as downloading files
+//! from HTTP, FTP, and S3 protocols.
+//!
 use crate::oneio::compressions::OneIOCompression;
 use crate::oneio::{compressions, get_writer_raw};
 use crate::OneIoError;
@@ -43,6 +46,10 @@ fn get_http_reader_raw(
             .as_str(),
         "true" | "yes" | "y" | "1"
     );
+    #[cfg(feature = "rustls")]
+    rustls_sys::crypto::aws_lc_rs::default_provider()
+        .install_default()
+        .ok();
 
     let client = match opt_client {
         Some(c) => c,
@@ -73,16 +80,24 @@ fn get_http_reader_raw(
     Ok(res)
 }
 
-/// Get a reader for remote content with the capability to specify headers, and customer reqwest options.
+/// Creates a reqwest blocking client with custom headers.
+///
+/// # Arguments
+///
+/// * `headers_map` - A argument of header key-value pairs.
+///
+/// # Returns
+///
+/// Returns a Result containing the constructed Client or a [OneIoError].
+///
+/// # Example
 ///
 /// Example usage with custom header fields:
 /// ```no_run
 /// use std::collections::HashMap;
 /// use reqwest::header::HeaderMap;
-/// let headers: HeaderMap = (&HashMap::from([("X-Custom-Auth-Key".to_string(), "TOKEN".to_string())])).try_into().expect("invalid headers");
-/// let client = reqwest::blocking::Client::builder()
-///        .default_headers(headers)
-///        .build().unwrap();
+///
+/// let client = oneio::create_client_with_headers([("X-Custom-Auth-Key", "TOKEN")]).unwrap();
 /// let mut reader = oneio::get_http_reader(
 ///   "https://SOME_REMOTE_RESOURCE_PROTECTED_BY_ACCESS_TOKEN",
 ///   Some(client),
@@ -91,6 +106,28 @@ fn get_http_reader_raw(
 /// reader.read_to_string(&mut text).unwrap();
 /// println!("{}", text);
 /// ```
+pub fn create_client_with_headers<I, K, V>(headers: I) -> Result<Client, OneIoError>
+where
+    I: IntoIterator<Item = (K, V)>,
+    K: Into<String>,
+    V: Into<String>,
+{
+    use reqwest::header::{HeaderMap, HeaderName, HeaderValue};
+    let mut header_map = HeaderMap::new();
+    for (k, v) in headers {
+        if let (Ok(name), Ok(value)) = (
+            HeaderName::from_bytes(k.into().as_bytes()),
+            HeaderValue::from_str(&v.into()),
+        ) {
+            header_map.insert(name, value);
+        }
+    }
+    Ok(Client::builder().default_headers(header_map).build()?)
+}
+
+/// Get a reader for remote content with the capability to specify headers, and customer reqwest options.
+///
+/// See [`create_client_with_headers`] for more details on how to create a client with custom headers.
 ///
 /// Example with customer builder that allows invalid certificates (bad practice):
 /// ```no_run
@@ -132,7 +169,7 @@ pub fn get_http_reader(
 ///
 /// * `remote_path` - The remote path of the file to download.
 /// * `local_path` - The local path where the downloaded file will be saved.
-/// * `header` - Optional header information to include in the request. If not specified, an empty HashMap should be provided.
+/// * `opt_client` - Optional custom [reqwest::blocking::Client] to use for the request.
 ///
 /// # Errors
 ///
@@ -194,7 +231,7 @@ pub fn download(
 ///
 /// * `remote_path` - The URL or file path of the file to download.
 /// * `local_path` - The file path to save the downloaded file.
-/// * `header` - Optional headers to include in the download request.
+/// * `opt_client` - Optional custom [reqwest::blocking::Client] to use for the request.
 /// * `retry` - The number of times to retry downloading in case of failure.
 ///
 /// # Errors
@@ -280,7 +317,11 @@ pub(crate) fn remote_file_exists(path: &str) -> Result<bool, OneIoError> {
     match get_protocol(path) {
         Some(protocol) => match protocol.as_str() {
             "http" | "https" => {
-                let client = reqwest::blocking::Client::builder()
+                #[cfg(feature = "rustls")]
+                rustls_sys::crypto::aws_lc_rs::default_provider()
+                    .install_default()
+                    .ok();
+                let client = Client::builder()
                     .timeout(std::time::Duration::from_secs(2))
                     .build()?;
                 let res = client.head(path).send()?;