Make async

2025-10-22 15:31:33 -04:00
parent abf7ceb882
commit b2bc77589e
3 changed files with 180 additions and 30 deletions
--- a/Cargo.lock
+++ b/Cargo.lock
@@ -47,6 +47,12 @@ dependencies = [
 "generic-array",
 ]

+[[package]]
+name = "bytes"
+version = "1.10.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d71b6127be86fdcfddb610f7182ac57211d4b18a3e9c82eb2d17662f2227ad6a"
+
 [[package]]
 name = "cfg-if"
 version = "1.0.4"
@@ -105,9 +111,11 @@ version = "0.1.0"
 dependencies = [
 "anyhow",
 "color-eyre",
+ "futures",
 "hex",
 "sha2",
 "thiserror",
+ "tokio",
 "tracing",
 ]

@@ -131,6 +139,95 @@ dependencies = [
 "once_cell",
 ]

+[[package]]
+name = "futures"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "65bc07b1a8bc7c85c5f2e110c476c7389b4554ba72af57d8445ea63a576b0876"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-executor",
+ "futures-io",
+ "futures-sink",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-channel"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2dff15bf788c671c1934e366d07e30c1814a8ef514e1af724a602e8a2fbe1b10"
+dependencies = [
+ "futures-core",
+ "futures-sink",
+]
+
+[[package]]
+name = "futures-core"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "05f29059c0c2090612e8d742178b0580d2dc940c837851ad723096f87af6663e"
+
+[[package]]
+name = "futures-executor"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1e28d1d997f585e54aebc3f97d39e72338912123a67330d723fdbb564d646c9f"
+dependencies = [
+ "futures-core",
+ "futures-task",
+ "futures-util",
+]
+
+[[package]]
+name = "futures-io"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9e5c1b78ca4aae1ac06c48a526a655760685149f0d465d21f37abfe57ce075c6"
+
+[[package]]
+name = "futures-macro"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "162ee34ebcb7c64a8abebc059ce0fee27c2262618d7b60ed8faf72fef13c3650"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "futures-sink"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "e575fab7d1e0dcb8d0c7bcf9a63ee213816ab51902e6d244a95819acacf1d4f7"
+
+[[package]]
+name = "futures-task"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f90f7dce0722e95104fcb095585910c0977252f286e354b5e3bd38902cd99988"
+
+[[package]]
+name = "futures-util"
+version = "0.3.31"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9fa08315bb612088cc391249efdc3bc77536f16c91f6cf495e6fbe85b20a4a81"
+dependencies = [
+ "futures-channel",
+ "futures-core",
+ "futures-io",
+ "futures-macro",
+ "futures-sink",
+ "futures-task",
+ "memchr",
+ "pin-project-lite",
+ "pin-utils",
+ "slab",
+]
+
 [[package]]
 name = "generic-array"
 version = "0.14.9"
@@ -213,6 +310,12 @@ version = "0.2.16"
 source = "registry+https://github.com/rust-lang/crates.io-index"
 checksum = "3b3cff922bd51709b605d9ead9aa71031d81447142d828eb4a6eba76fe619f9b"

+[[package]]
+name = "pin-utils"
+version = "0.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8b870d8c151b6f2fb93e84a13146138f05d02ed11c7e7c54f8826aaaf7c9f184"
+
 [[package]]
 name = "proc-macro2"
 version = "1.0.101"
@@ -257,6 +360,12 @@ dependencies = [
 "lazy_static",
 ]

+[[package]]
+name = "slab"
+version = "0.4.11"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7a2ae44ef20feb57a68b23d846850f861394c2e02dc425a50098ae8c90267589"
+
 [[package]]
 name = "syn"
 version = "2.0.107"
@@ -297,6 +406,28 @@ dependencies = [
 "cfg-if",
 ]

+[[package]]
+name = "tokio"
+version = "1.48.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ff360e02eab121e0bc37a2d3b4d4dc622e6eda3a8e5253d5435ecf5bd4c68408"
+dependencies = [
+ "bytes",
+ "pin-project-lite",
+ "tokio-macros",
+]
+
+[[package]]
+name = "tokio-macros"
+version = "2.6.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af407857209536a95c8e56f8231ef2c2e2aff839b22e07a1ffcbc617e9db9fa5"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
 [[package]]
 name = "tracing"
 version = "0.1.41"
--- a/Cargo.toml
+++ b/Cargo.toml
@@ -6,7 +6,9 @@ edition = "2024"
 [dependencies]
 anyhow = "1.0.100"
 color-eyre = "0.6.5"
+futures = "0.3.31"
 hex = "0.4.3"
 sha2 = "0.10.9"
 thiserror = "2.0.17"
+tokio = { version = "1.48.0", features = ["rt-multi-thread", "macros", "fs", "io-std", "io-util"] }
 tracing = "0.1.41"
--- a/src/main.rs
+++ b/src/main.rs
@@ -1,39 +1,46 @@
+use futures::{StreamExt, stream};
 use std::{
    collections::VecDeque,
-    fs::{DirEntry, File, ReadDir},
+    fs::File,
    io::{BufReader, Read},
    path::{Path, PathBuf},
 };

+use tokio::{fs, task};
+
 use anyhow::Result;
 use sha2::{Digest, Sha256};

 const BUF_SIZE: usize = 256 * 1024;

-fn hash_file(path: &Path) -> Result<String> {
-    let mut hasher = Sha256::new();
-    let file = File::open(path)?;
-    let mut reader = BufReader::with_capacity(BUF_SIZE, file);
-    let mut buffer = vec![0u8; BUF_SIZE];
-    let mut n = 1;
-    while n > 0 {
-        n = reader.read(&mut buffer)?;
-        hasher.update(&buffer[..n]);
-    }
-    let hash = hasher.finalize();
+async fn hash_file(path: PathBuf) -> Result<String> {
+    task::spawn_blocking(move || -> Result<String> {
+        let file = File::open(&path)?;
+        let mut reader = BufReader::with_capacity(BUF_SIZE, file);
+        let mut hasher = Sha256::new();
+        let mut buffer = vec![0u8; BUF_SIZE];

-    Ok(hex::encode(hash))
+        loop {
+            let n = reader.read(&mut buffer)?;
+            if n == 0 {
+                break;
+            }
+            hasher.update(&buffer[..n]);
+        }
+
+        Ok(hex::encode(hasher.finalize()))
+    })
+    .await?
 }

-pub fn collect_file_paths(root: PathBuf) -> Result<Vec<PathBuf>> {
-    let mut paths: Vec<PathBuf> = Vec::new();
-    let mut dirs: VecDeque<PathBuf> = VecDeque::with_capacity(16);
+async fn collect_file_paths(root: PathBuf) -> Result<Vec<PathBuf>> {
+    let mut paths = Vec::new();
+    let mut dirs = VecDeque::with_capacity(16);
    dirs.push_back(root);

    while let Some(directory) = dirs.pop_front() {
-        let dir_entry = std::fs::read_dir(directory)?;
-        for entry in dir_entry {
-            let entry = entry?;
+        let mut dir_entry = tokio::fs::read_dir(directory).await?;
+        while let Some(entry) = dir_entry.next_entry().await? {
            let path = entry.path();

            if path.is_dir() {
@@ -47,19 +54,29 @@ pub fn collect_file_paths(root: PathBuf) -> Result<Vec<PathBuf>> {
    Ok(paths)
 }

-fn main() -> Result<()> {
+#[tokio::main]
+async fn main() -> Result<()> {
    let root = ".";
-    let files = collect_file_paths(root.into())?;
+    let files = collect_file_paths(root.into()).await?;
+    let cores = 64;

-    for path in files {
-        let file = path.file_name().unwrap();
-        let hash = hash_file(&path)?;
-        println!(
-            "Got Hash: {hash}, File: {:?}, Path: {:?}",
-            file,
-            path.canonicalize().unwrap()
-        );
-    }
+    stream::iter(files)
+        .map(|path| async move {
+            let name = path.file_name().map(|s| s.to_owned());
+            let canonical = fs::canonicalize(&path).await.unwrap_or(path.clone());
+            let hash = hash_file(path).await;
+            (hash, name, canonical)
+        })
+        .buffer_unordered(cores)
+        .for_each(|(hash, name, path)| async move {
+            println!(
+                "Got Hash: {}, File: {:?}, Path: {:?}",
+                hash.unwrap(),
+                name,
+                path
+            );
+        })
+        .await;

    Ok(())
 }