parallelize set operations

Author: KonradHoeffner

benches/criterion.rs

@@ -92,7 +92,7 @@ fn query(c: &mut Criterion) {
 fn read_nt_benchmarks(c: &mut Criterion) {
     let mut group = c.benchmark_group("dictionary_read_nt");
     group.sample_size(10);
-    let test_file = "tests/resources/tax.nt";
+    let test_file = "tests/resources/persondata_en.nt";
 
     // Benchmark 1: N-Triples parsing
     group.bench_function("nt_parsing", |b| {

src/four_sect_dict.rs

@@ -213,19 +213,29 @@ impl FourSectDict {
         block_size: usize,
     ) -> Self {
         use log::warn;
-
         if predicate_terms.is_empty() {
             warn!("no triples found in provided RDF");
         }
         let predicate_terms_ref: BTreeSet<&str> = predicate_terms.iter().map(std::ops::Deref::deref).collect();
 
-        let shared_terms: BTreeSet<&str> =
-            subject_terms.intersection(object_terms).map(std::ops::Deref::deref).collect();
-        let unique_subject_terms: BTreeSet<&str> =
-            subject_terms.difference(object_terms).map(std::ops::Deref::deref).collect();
-        let unique_object_terms: BTreeSet<&str> =
-            object_terms.difference(subject_terms).map(std::ops::Deref::deref).collect();
+        let [shared_terms, unique_subject_terms, unique_object_terms]: [BTreeSet<&str>; 3] =
+            std::thread::scope(|s| {
+                [
+                    s.spawn(|| subject_terms.intersection(object_terms).map(std::ops::Deref::deref).collect()),
+                    s.spawn(|| subject_terms.difference(object_terms).map(std::ops::Deref::deref).collect()),
+                    s.spawn(|| object_terms.difference(subject_terms).map(std::ops::Deref::deref).collect()),
+                ]
+                .map(|t| t.join().unwrap())
+            });
+
+        let (shared, subjects, predicates, objects) = (
+            DictSectPFC::compress(&shared_terms, block_size),
+            DictSectPFC::compress(&unique_subject_terms, block_size),
+            DictSectPFC::compress(&predicate_terms_ref, block_size),
+            DictSectPFC::compress(&unique_object_terms, block_size),
+        );
 
+        /*
         // Parallelize dictionary compression using rayon
         let ((shared, predicates), (subjects, objects)) = rayon::join(
             || {
@@ -241,6 +251,15 @@ impl FourSectDict {
                 )
             },
         );
+        */
+        /*
+        use std::thread;
+        let [shared, predicates, subjects, objects] = thread::scope(|s| {
+            [&shared_terms, &predicate_terms_ref, &unique_subject_terms, &unique_object_terms]
+                .map(|terms| s.spawn(|| DictSectPFC::compress(terms, block_size)))
+                .map(|t| t.join().unwrap())
+        });
+        */
 
         FourSectDict { shared, subjects, predicates, objects }
     }