Changes to set max sample size

Ignore address frequency
Be more permissive matching strings
Vendored
Support zero based images

Author: Your Name

.cargo/config.toml

@@ -0,0 +1,5 @@
+[source.crates-io]
+replace-with = "vendored-sources"
+
+[source.vendored-sources]
+directory = "vendor"

Cargo.lock

@@ -1,6 +1,6 @@
 [package]
 name = "rbase"
-version = "0.1.6"
+version = "0.1.7"
 edition = "2021"
 repository = "https://github.com/WorksButNotTested/rbase.git"
 authors = [ "WorksButNotTested" ]

Cargo.toml

@@ -92,6 +92,22 @@ pub struct Args {
         default_value = "8"
     )]
     pub jobs: usize,
+
+    #[arg(
+        short = 's',
+        long = "max-strings",
+        help = "Maximum number of strings to sample",
+        default_value = "100000"
+    )]
+    pub max_strings: usize,
+
+    #[arg(
+        short = 'a',
+        long = "max-addresses",
+        help = "Maximum number of addresses to sample",
+        default_value = "1000000"
+    )]
+    pub max_addresses: usize,
 }
 
 impl Args {
@@ -169,7 +185,7 @@ impl<
     }
 
     fn get_string_offsets(min: usize, max: usize, chunks: Vec<(usize, &[u8])>) -> DashSet<T> {
-        let regex = format!("([a-zA-Z0-9_]{{{},{}}})\0", min, max);
+        let regex = format!("([[:print:][:space:]]{{{},{}}})\0", min, max);
         let re = Regex::new(&regex).unwrap();
         let pb = Self::get_progress_bar("Finding strings", chunks.len());
         let set = DashSet::<T>::new();
@@ -182,14 +198,13 @@ impl<
                     set.insert(k);
                 });
             });
-        println!("Found: {:?} strings", set.len());
         set
     }
 
-    fn index_strings_by_page_offset(addresses: DashSet<T>) -> DashMap<T, Vec<T>> {
-        let pb = Self::get_progress_bar("Indexing strings", addresses.len());
+    fn index_strings_by_page_offset(strings: DashSet<T>) -> DashMap<T, Vec<T>> {
+        let pb = Self::get_progress_bar("Indexing strings", strings.len());
         let map = DashMap::<T, Vec<T>>::new();
-        addresses.into_par_iter().progress_with(pb).for_each(|k| {
+        strings.into_par_iter().progress_with(pb).for_each(|k| {
             let offset = k & T::try_from(PAGE_OFFSET_MASK).unwrap();
             if let Some(mut v) = map.get_mut(&offset) {
                 v.push(k);
@@ -203,37 +218,40 @@ impl<
     pub fn get_strings_by_page_offset(min: usize, max: usize, bytes: &[u8]) -> DashMap<T, Vec<T>> {
         let chunks = Self::get_overlapping_chunks(bytes, max - 1);
         let addresses = Self::get_string_offsets(min, max, chunks);
-        println!("Found: {:?} unique strings", addresses.len());
+        println!("Found: {:?} strings", addresses.len());
         let index = Self::index_strings_by_page_offset(addresses);
         index
     }
 
     /* Addresses */
-    fn get_address_frequencies<F: Fn(&[u8]) -> T + Sync + Send>(
+    fn get_addresses<F: Fn(&[u8]) -> T + Sync + Send>(
         bytes: &[u8],
         convert: F,
-    ) -> DashMap<T, usize> {
+    ) -> DashSet<T> {
         let chunks = bytes.chunks(size_of::<T>()).collect::<Vec<&[u8]>>();
         let pb = Self::get_progress_bar("Reading addresses", chunks.len());
-        let map = DashMap::<T, usize>::new();
+        let set = DashSet::<T>::new();
         chunks
             .into_par_iter()
             .progress_with(pb)
             .map(|p| convert(p))
             .filter(|&p| p != T::default())
             .for_each(|ptr| {
-                *map.entry(ptr).or_insert(0) += 1;
+                set.insert(ptr);
             });
-        map
+        set
     }
 
-    fn index_unique_addresses_by_page_offset(frequencies: DashMap<T, usize>) -> DashMap<T, Vec<T>> {
+    fn index_addresses_by_page_offset(
+        addresses: DashSet<T>,
+        max_addresses: usize,
+    ) -> DashMap<T, Vec<T>> {
         let map = DashMap::<T, Vec<T>>::new();
-        let pb = Self::get_progress_bar("Finding unique addresses", frequencies.len());
-        frequencies
+        let pb = Self::get_progress_bar("Indexing addresses", addresses.len());
+        addresses
             .into_par_iter()
+            .take_any(max_addresses)
             .progress_with(pb)
-            .filter_map(|(k, v)| if v > 1 { Some(k) } else { None })
             .for_each(|k| {
                 let offset = k & T::try_from(PAGE_OFFSET_MASK).unwrap();
                 if let Some(mut v) = map.get_mut(&offset) {
@@ -248,13 +266,13 @@ impl<
     pub fn get_addresses_by_page_offset<F: Fn(&[u8]) -> T + Sync + Send + Copy>(
         bytes: &[u8],
         convert: F,
+        max_addresses: usize,
     ) -> DashMap<T, Vec<T>> {
-        let frequencies = Self::get_address_frequencies(bytes, convert);
-        println!("Found: {:?} addresses", frequencies.len());
+        let addresses = Self::get_addresses(bytes, convert);
+        println!("Found: {:?} addresses", addresses.len());
 
-        let addresses = Self::index_unique_addresses_by_page_offset(frequencies);
-        println!("Found: {:?} unique addresses", addresses.len());
-        addresses
+        let index = Self::index_addresses_by_page_offset(addresses, max_addresses);
+        index
     }
 
     /* Addresses */
@@ -268,7 +286,7 @@ impl<
             let (offset, strings) = r.pair();
             if let Some(addresses) = addresses.get(offset) {
                 for &s in strings.iter() {
-                    for &a in addresses.iter().filter(|&&a| a > s) {
+                    for &a in addresses.iter().filter(|&&a| a >= s) {
                         *map.entry(a - s).or_insert(0) += 1;
                     }
                 }
@@ -290,7 +308,7 @@ impl<
     pub fn get_most_frequent_candidate_base_address(
         strings: &DashMap<T, Vec<T>>,
         addresses: &DashMap<T, Vec<T>>,
-    ) -> T {
+    ) -> Option<T> {
         let base_addresses = Self::get_candidate_base_addresses(strings, addresses);
         let num_candidates = base_addresses.len();
         println!("Found: {:?} candidates", num_candidates);
@@ -301,11 +319,11 @@ impl<
         let sorted = Self::sort_candidate_base_addresses_by_frequency(filtered);
         for (idx, (base, frequency)) in sorted.iter().take(10).enumerate() {
             let pct = 100.0 * (*frequency as f64) / (num_candidates as f64);
-            println!("{:2}: {base:x}: {frequency} ({pct:.2}%)", idx + 1);
+            println!("{:2}: {base:0x}: {frequency} ({pct:.2}%)", idx + 1);
         }
 
-        let (base, _frequency) = sorted.first().unwrap().clone();
-        base
+        let (base, _frequency) = sorted.first().cloned()?;
+        Some(base)
     }
 }
 
@@ -328,9 +346,15 @@ fn main() {
                     Endian::Little => |bytes: &[u8]| u32::from_le_bytes(bytes.try_into().unwrap()),
                     Endian::Big => |bytes: &[u8]| u32::from_be_bytes(bytes.try_into().unwrap()),
                 },
+                args.max_addresses,
             );
-            let base = RBase::get_most_frequent_candidate_base_address(&strings, &addresses);
-            println!("Found base: {:x}", base);
+            if let Some(base) =
+                RBase::get_most_frequent_candidate_base_address(&strings, &addresses)
+            {
+                println!("Found base: {:0x}", base);
+            } else {
+                println!("No base found");
+            }
         }
         Size::Bits64 => {
             let strings = RBase::get_strings_by_page_offset(args.min, args.max, bytes);
@@ -340,9 +364,15 @@ fn main() {
                     Endian::Little => |bytes: &[u8]| u64::from_le_bytes(bytes.try_into().unwrap()),
                     Endian::Big => |bytes: &[u8]| u64::from_be_bytes(bytes.try_into().unwrap()),
                 },
+                args.max_addresses,
             );
-            let base = RBase::get_most_frequent_candidate_base_address(&strings, &addresses);
-            println!("Found base: {:x}", base);
+            if let Some(base) =
+                RBase::get_most_frequent_candidate_base_address(&strings, &addresses)
+            {
+                println!("Found base: {:x}", base);
+            } else {
+                println!("No base found");
+            }
         }
     };
     let end = start.elapsed();

src/main.rs

@@ -0,0 +1 @@
+{"files":{"COPYING":"01c266bced4a434da0051174d6bee16a4c82cf634e2679b6155d40d75012390f","Cargo.toml":"88c12a803c6c06c47cd9dabc8bcdba81f35d3bab637221d2106a86a543532731","DESIGN.md":"59c960e1b73b1d7fb41e4df6c0c1b1fcf44dd2ebc8a349597a7d0595f8cb5130","LICENSE-MIT":"0f96a83840e146e43c0ec96a22ec1f392e0680e6c1226e6f3ba87e0740af850f","README.md":"afc4d559a98cf190029af0bf320fc0022725e349cd2a303aac860254e28f3c53","UNLICENSE":"7e12e5df4bae12cb21581ba157ced20e1986a0508dd10d0e8a4ab9a4cf94e85c","rustfmt.toml":"1ca600239a27401c4a43f363cf3f38183a212affc1f31bff3ae93234bbaec228","src/ahocorasick.rs":"c699c07df70be45c666e128509ad571a7649d2073e4ae16ac1efd6793c9c6890","src/automaton.rs":"22258a3e118672413119f8f543a9b912cce954e63524575c0ebfdf9011f9c2dd","src/dfa.rs":"bfef1a94c5e7410584b1beb4e857b40d1ae2031b881cbc06fb1300409bbd555f","src/lib.rs":"2a92d5c5e930f2d306508802e8a929135e1f41c9f5f8deda8f7eb98947179dd2","src/macros.rs":"c6c52ae05b24433cffaca7b78b3645d797862c5d5feffddf9f54909095ed6e05","src/nfa/contiguous.rs":"aeb6ee5fd80eea04decbc4b46aa27d1ab270b78d416a644da25b7934f009ee66","src/nfa/mod.rs":"ee7b3109774d14bbad5239c16bb980dd6b8185ec136d94fbaf2f0dc27d5ffa15","src/nfa/noncontiguous.rs":"de94f02b04efd8744fb096759a8897c22012b0e0ca3ace161fd87c71befefe04","src/packed/api.rs":"160d3b10823316f7b0924e13c3afd222c8a7db5c0a00432401f311ef27d6a1b7","src/packed/ext.rs":"66be06fde8558429da23a290584d4b9fae665bf64c2578db4fe5f5f3ee864869","src/packed/mod.rs":"0020cd6f07ba5c8955923a9516d7f758864260eda53a6b6f629131c45ddeec62","src/packed/pattern.rs":"1e3a289a730c141fc30b295811e372d046c6619c7fd670308299b889a06c7673","src/packed/rabinkarp.rs":"403146eb1d838a84601d171393542340513cd1ee7ff750f2372161dd47746586","src/packed/teddy/README.md":"3a43194b64e221543d885176aba3beb1224a927385a20eca842daf6b0ea2f342","src/packed/teddy/builder.rs":"08ec116a4a842a2bb1221d296a2515ef3672c54906bed588fb733364c07855d3","src/packed/teddy/generic.rs":"ea252ab05b32cea7dd9d71e332071d243db7dd0362e049252a27e5881ba2bf39","src/packed/teddy/mod.rs":"17d741f7e2fb9dbac5ba7d1bd4542cf1e35e9f146ace728e23fe6bbed20028b2","src/packed/tests.rs":"8e2f56eb3890ed3876ecb47d3121996e416563127b6430110d7b516df3f83b4b","src/packed/vector.rs":"70c325cfa6f7c5c4c9a6af7b133b75a29e65990a7fe0b9a4c4ce3c3d5a0fe587","src/tests.rs":"c68192ab97b6161d0d6ee96fefd80cc7d14e4486ddcd8d1f82b5c92432c24ed5","src/transducer.rs":"02daa33a5d6dac41dcfd67f51df7c0d4a91c5131c781fb54c4de3520c585a6e1","src/util/alphabet.rs":"6dc22658a38deddc0279892035b18870d4585069e35ba7c7e649a24509acfbcc","src/util/buffer.rs":"f9e37f662c46c6ecd734458dedbe76c3bb0e84a93b6b0117c0d4ad3042413891","src/util/byte_frequencies.rs":"2fb85b381c038c1e44ce94294531cdcd339dca48b1e61f41455666e802cbbc9e","src/util/debug.rs":"ab301ad59aa912529cb97233a54a05914dd3cb2ec43e6fec7334170b97ac5998","src/util/error.rs":"ecccd60e7406305023efcc6adcc826eeeb083ab8f7fbfe3d97469438cd4c4e5c","src/util/int.rs":"e264e6abebf5622b59f6500210773db36048371c4e509c930263334095959a52","src/util/mod.rs":"7ab28d11323ecdbd982087f32eb8bceeee84f1a2583f3aae27039c36d58cf12c","src/util/prefilter.rs":"9fa4498f18bf70478b1996c1a013698b626d15f119aa81dbc536673c9f045718","src/util/primitives.rs":"f89f3fa1d8db4e37de9ca767c6d05e346404837cade6d063bba68972fafa610b","src/util/remapper.rs":"9f12d911583a325c11806eeceb46d0dfec863cfcfa241aed84d31af73da746e5","src/util/search.rs":"6af803e08b8b8c8a33db100623f1621b0d741616524ce40893d8316897f27ffe","src/util/special.rs":"7d2f9cb9dd9771f59816e829b2d96b1239996f32939ba98764e121696c52b146"},"package":"8e60d3430d3a69478ad0993f19238d2df97c507009a52b3c10addcd7f6bcb916"}

vendor/aho-corasick/.cargo-checksum.json

@@ -0,0 +1,3 @@
+This project is dual-licensed under the Unlicense and MIT licenses.
+
+You may use this code under the terms of either license.

vendor/aho-corasick/COPYING

@@ -0,0 +1,74 @@
+# THIS FILE IS AUTOMATICALLY GENERATED BY CARGO
+#
+# When uploading crates to the registry Cargo will automatically
+# "normalize" Cargo.toml files for maximal compatibility
+# with all versions of Cargo and also rewrite `path` dependencies
+# to registry (e.g., crates.io) dependencies.
+#
+# If you are reading this file be aware that the original Cargo.toml
+# will likely look very different (and much more reasonable).
+# See Cargo.toml.orig for the original contents.
+
+[package]
+edition = "2021"
+rust-version = "1.60.0"
+name = "aho-corasick"
+version = "1.1.3"
+authors = ["Andrew Gallant <[email protected]>"]
+exclude = [
+    "/aho-corasick-debug",
+    "/benchmarks",
+    "/tmp",
+]
+autotests = false
+description = "Fast multiple substring searching."
+homepage = "https://github.com/BurntSushi/aho-corasick"
+readme = "README.md"
+keywords = [
+    "string",
+    "search",
+    "text",
+    "pattern",
+    "multi",
+]
+categories = ["text-processing"]
+license = "Unlicense OR MIT"
+repository = "https://github.com/BurntSushi/aho-corasick"
+
+[package.metadata.docs.rs]
+all-features = true
+rustdoc-args = [
+    "--cfg",
+    "docsrs",
+    "--generate-link-to-definition",
+]
+
+[profile.bench]
+debug = 2
+
+[profile.release]
+debug = 2
+
+[lib]
+name = "aho_corasick"
+
+[dependencies.log]
+version = "0.4.17"
+optional = true
+
+[dependencies.memchr]
+version = "2.4.0"
+optional = true
+default-features = false
+
+[dev-dependencies.doc-comment]
+version = "0.3.3"
+
+[features]
+default = [
+    "std",
+    "perf-literal",
+]
+logging = ["dep:log"]
+perf-literal = ["dep:memchr"]
+std = ["memchr?/std"]