remove leftover info logs. update HDT type support comments. make it easier to write a cache file (#92)

Author: GregHanson

README.md

@@ -11,7 +11,7 @@
 
 A Rust library for the [Header Dictionary Triples](https://www.rdfhdt.org/) compressed RDF format, including:
 
-* loading the HDT default format as created by [hdt-cpp](https://github.com/rdfhdt/hdt-cpp)
+* loading the HDT default format as created by this library or [hdt-cpp](https://github.com/rdfhdt/hdt-cpp)
 * converting N-Triples to HDT
 * efficient querying by triple patterns
 * serializing into other formats like RDF Turtle and N-Triples using the [Sophia](https://crates.io/crates/sophia) adapter

src/hdt.rs

@@ -67,7 +67,7 @@ impl Hdt {
     }
 
     /// Creates an immutable HDT instance containing the dictionary and triples from the given reader.
-    /// The reader must point to the beginning of the data of an HDT file as produced by hdt-cpp.
+    /// The reader must point to the beginning of the data of an HDT file.
     /// FourSectionDictionary with DictionarySectionPlainFrontCoding and SPO order is the only supported implementation.
     /// The format is specified at <https://www.rdfhdt.org/hdt-binary-format/>, however there are some deviations.
     /// The initial HDT specification at <http://www.w3.org/Submission/2011/03/> is outdated and not supported.
@@ -99,7 +99,7 @@ impl Hdt {
 
     /// Creates an immutable HDT instance containing the dictionary and triples from the Path.
     /// Will utilize a custom cached TriplesBitmap file if exists or create one if it does not exist.
-    /// The file path must point to the beginning of the data of an HDT file as produced by hdt-cpp.
+    /// The file path must point to the beginning of the data of an HDT file.
     /// FourSectionDictionary with DictionarySectionPlainFrontCoding and SPO order is the only supported implementation.
     /// The format is specified at <https://www.rdfhdt.org/hdt-binary-format/>, however there are some deviations.
     /// The initial HDT specification at <http://www.w3.org/Submission/2011/03/> is outdated and not supported.
@@ -179,7 +179,9 @@ impl Hdt {
     }
 
     #[cfg(feature = "cache")]
-    fn write_cache(
+    /// Writes a custom cache file to improve load times. This cache file is usuable only by
+    /// this library and is not intended to be used with hdt-cpp or hdt-java versions of the HDT tooling
+    pub fn write_cache(
         index_file_path: &std::path::PathBuf, triples: &TriplesBitmap, header_length: usize,
     ) -> core::result::Result<(), Box<dyn std::error::Error>> {
         let new_index_file = File::create(index_file_path)?;

src/lib.rs

@@ -9,7 +9,7 @@
 //!
 //! HDT is a loading and triple pattern querying library for the [Header Dictionary Triples](https://www.rdfhdt.org/) compressed binary RDF format.
 //!
-//! Currently this library only supports loading and querying existing HDT files as created by [hdt-cpp](https://github.com/rdfhdt/hdt-cpp).
+//! Currently this library only supports loading and querying existing HDT files as created by this library or [hdt-cpp](https://github.com/rdfhdt/hdt-cpp).
 //! For reference implementations of HDT in C++ and Java, which support conversion and serialization from and into HDT with different format options,
 //! and acknowledgement of all the original authors, please look at the <https://github.com/rdfhdt> organisation.
 //!

src/nt.rs

@@ -6,14 +6,13 @@ use crate::{DictSectPFC, FourSectDict, IdKind};
 use bitset_core::BitSet;
 use bytesize::ByteSize;
 use lasso::{Key, Spur, ThreadedRodeo};
-use log::{debug, error, info};
+use log::{debug, error};
 use oxttl::NTriplesParser;
 use rayon::prelude::*;
 use std::collections::BTreeSet;
 use std::path::Path;
 use std::sync::Arc;
 use std::thread;
-use std::time::Instant;
 
 pub type Result<T> = std::io::Result<T>;
 type Simd = [u64; 4];
@@ -126,9 +125,7 @@ struct IndexPool {
 /// read N-Triples and convert them to a dictionary and triple IDs
 fn read_dict_triples(path: &Path, block_size: usize) -> Result<(FourSectDict, Vec<TripleId>)> {
     // 1. Parse N-Triples and collect terms using string interning
-    let timer = Instant::now();
     let mut pool = parse_nt_terms(path)?;
-    let parse_time = timer.elapsed();
 
     // Sort and deduplicate triples in parallel with dictionary building
     let mut triples = std::mem::take(&mut pool.triples); // not needed anymore
@@ -139,12 +136,9 @@ fn read_dict_triples(path: &Path, block_size: usize) -> Result<(FourSectDict, Ve
     })?;
 
     // 2. Build dictionary from term indices
-    let timer = Instant::now();
     let dict = build_dict_from_terms(&pool, block_size);
-    let dict_build_time = timer.elapsed();
 
     // 3. Encode triples to IDs using dictionary
-    let timer = Instant::now();
     let sorted_triple_indices = sorter.join().unwrap();
     let refs: &[[usize; 3]] = &sorted_triple_indices;
     let encoded_triples: Vec<TripleId> = refs
@@ -165,8 +159,6 @@ fn read_dict_triples(path: &Path, block_size: usize) -> Result<(FourSectDict, Ve
         })
         .collect();
 
-    info!("{parse_time:?},{dict_build_time:?},{:?}", timer.elapsed());
-
     Ok((dict, encoded_triples))
 }