diff --git a/Cargo.lock b/Cargo.lock index c78c3d80fa..059ec6986d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -41,6 +41,12 @@ dependencies = [ "alloc-no-stdlib", ] +[[package]] +name = "allocator-api2" +version = "0.2.21" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "683d7910e743518b0e34f1186f92494becacb047c7b6bf616c96772180fef923" + [[package]] name = "android-tzdata" version = "0.1.1" @@ -630,6 +636,15 @@ version = "2.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "5c8214115b7bf84099f1309324e63141d4c5d7cc26862f97a0a857dbefe165bd" +[[package]] +name = "bitpacking" +version = "0.9.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4c1d3e2bfd8d06048a179f7b17afc3188effa10385e7b00dc65af6aae732ea92" +dependencies = [ + "crunchy", +] + [[package]] name = "block-buffer" version = "0.10.4" @@ -732,6 +747,12 @@ dependencies = [ "shlex", ] +[[package]] +name = "census" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4f4c707c6a209cbe82d10abd08e1ea8995e9ea937d2550646e02798948992be0" + [[package]] name = "cexpr" version = "0.6.0" @@ -968,6 +989,15 @@ dependencies = [ "itertools 0.10.5", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.14" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "06ba6d68e24814cb8de6bb986db8222d3a027d15872cabc0d18817bc3c0e4471" +dependencies = [ + "crossbeam-utils", +] + [[package]] name = "crossbeam-deque" version = "0.8.6" @@ -1193,6 +1223,12 @@ dependencies = [ "syn 2.0.100", ] +[[package]] +name = "downcast-rs" +version = "1.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75b325c5dbd37f80359721ad39aca5a29fb04c89279657cffdda8736d0c0b9d2" + [[package]] name = "dunce" version = "1.0.5" @@ -1286,6 +1322,12 @@ version = "1.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3ebc5a6d89e3c90b84e8f33c8737933dda8f1c106b5415900b38b9d433841478" +[[package]] +name = "fastdivide" +version = "0.4.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "9afc2bd4d5a73106dd53d10d73d3401c2f32730ba2c0b93ddb888a8983680471" + [[package]] name = "fastrand" version = "2.3.0" @@ -1308,6 +1350,12 @@ version = "1.0.7" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "3f9eec918d3f24069decb9af1554cad7c880e2da24a9afd88aca000531ab82c1" +[[package]] +name = "foldhash" +version = "0.1.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d9c4f5dac5e15c24eb999c26181a6ca40b39fe946cbe4c263c7209467bc83af2" + [[package]] name = "foreign-types" version = "0.3.2" @@ -1342,6 +1390,16 @@ dependencies = [ "tokio", ] +[[package]] +name = "fs4" +version = "0.8.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f7e180ac76c23b45e767bd7ae9579bc0bb458618c4bc71835926e098e61d15f8" +dependencies = [ + "rustix 0.38.44", + "windows-sys 0.52.0", +] + [[package]] name = "fs_extra" version = "1.3.0" @@ -1559,6 +1617,11 @@ name = "hashbrown" version = "0.15.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "bf151400ff0baff5465007dd2f3e717f3fe502074ca563069ce3a6629d07b289" +dependencies = [ + "allocator-api2", + "equivalent", + "foldhash", +] [[package]] name = "heck" @@ -1566,6 +1629,12 @@ version = "0.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "2304e00983f87ffb38b55b444b5e3b60a884b5d30c0fca7d82fe33449bbe55ea" +[[package]] +name = "hermit-abi" +version = "0.3.9" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d231dfb89cfffdbc30e7fc41579ed6066ad03abda9e567ccafae602b97ec5024" + [[package]] name = "hermit-abi" version = "0.4.0" @@ -1620,6 +1689,12 @@ version = "0.2.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "459a0ca33ee92551e0a3bb1774f2d3bdd1c09fb6341845736662dd25e1fcb52a" +[[package]] +name = "htmlescape" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e9025058dae765dee5070ec375f591e2ba14638c63feff74f13805a72e523163" + [[package]] name = "http" version = "0.2.12" @@ -2002,6 +2077,9 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "e0242819d153cba4b4b05a5a8f2a7e9bbf97b6055b2a002b395c96b5ff3c0222" dependencies = [ "cfg-if 1.0.0", + "js-sys", + "wasm-bindgen", + "web-sys", ] [[package]] @@ -2188,6 +2266,12 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "830d08ce1d1d941e6b30645f1a0eb5643013d835ce3779a5fc208261dbe10f55" +[[package]] +name = "levenshtein_automata" +version = "0.2.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "0c2cdeb66e45e9f36bfad5bbdb4d2384e70936afbee843c6f6543f0c551ebb25" + [[package]] name = "libc" version = "0.2.171" @@ -2204,6 +2288,12 @@ dependencies = [ "windows-targets 0.52.6", ] +[[package]] +name = "libm" +version = "0.2.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8355be11b20d696c8f18f6cc018c4e372165b1fa8126cef092399c9951984ffa" + [[package]] name = "libredox" version = "0.1.3" @@ -2248,6 +2338,21 @@ version = "0.4.27" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "13dc2df351e3202783a1fe0d44375f7295ffb4049267b0f3018346dc122a1d94" +[[package]] +name = "lru" +version = "0.12.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "234cf4f4a04dc1f57e24b96cc0cd600cf2af460d4161ac5ecdd0af8e1f3b2a38" +dependencies = [ + "hashbrown 0.15.2", +] + +[[package]] +name = "lz4_flex" +version = "0.11.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "75761162ae2b0e580d7e7c390558127e5f01b4194debd6221fd8c207fc80e3f5" + [[package]] name = "matchit" version = "0.8.4" @@ -2260,12 +2365,31 @@ version = "2.0.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "60302e4db3a61da70c0cb7991976248362f30319e88850c487b9b95bbf059e00" +[[package]] +name = "measure_time" +version = "0.8.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "dbefd235b0aadd181626f281e1d684e116972988c14c264e42069d5e8a5775cc" +dependencies = [ + "instant", + "log", +] + [[package]] name = "memchr" version = "2.7.4" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "78ca9ab1a0babb1e7d5695e3530886289c18cf2f87ec19a575a0abdce112e3a3" +[[package]] +name = "memmap2" +version = "0.9.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "fd3f7eed9d3848f8b98834af67102b720745c4ec028fcd0aa0239277e7de374f" +dependencies = [ + "libc", +] + [[package]] name = "mime" version = "0.3.17" @@ -2381,6 +2505,12 @@ dependencies = [ "thiserror 1.0.69", ] +[[package]] +name = "murmurhash32" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2195bf6aa996a481483b29d62a7663eed3fe39600c460e323f8ff41e90bdd89b" + [[package]] name = "native-tls" version = "0.2.14" @@ -2506,6 +2636,17 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "071dfc062690e90b734c0b2273ce72ad0ffa95f0c74596bc250dcfd960262841" dependencies = [ "autocfg", + "libm", +] + +[[package]] +name = "num_cpus" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "4161fcb6d602d4d2081af7c3a45852d875a03dd337a6bfdd6e06407b61342a43" +dependencies = [ + "hermit-abi 0.3.9", + "libc", ] [[package]] @@ -2547,6 +2688,12 @@ version = "1.21.3" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "42f5e15c9953c5e4ccceeb2e7382a716482c34515315f7b03532b8b4e8393d2d" +[[package]] +name = "oneshot" +version = "0.1.11" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b4ce411919553d3f9fa53a0880544cda985a112117a0444d5ff1e870a893d6ea" + [[package]] name = "oorandom" version = "11.1.5" @@ -2663,6 +2810,7 @@ dependencies = [ "sha3", "snafu", "sysinfo", + "tantivy", "tempfile", "tokio", "tokio-stream", @@ -2709,6 +2857,15 @@ dependencies = [ "thiserror 2.0.12", ] +[[package]] +name = "ownedbytes" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c3a059efb063b8f425b948e042e6b9bd85edfe60e913630ed727b23e2dfcc558" +dependencies = [ + "stable_deref_trait", +] + [[package]] name = "parking" version = "2.2.1" @@ -2951,6 +3108,16 @@ dependencies = [ "getrandom 0.2.15", ] +[[package]] +name = "rand_distr" +version = "0.4.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "32cb0b9bc82b0a0876c2dd994a7e7a2683d3e7390ca40e6886785ef0c7e3ee31" +dependencies = [ + "num-traits", + "rand", +] + [[package]] name = "rayon" version = "1.10.0" @@ -3169,6 +3336,16 @@ dependencies = [ "walkdir", ] +[[package]] +name = "rust-stemmers" +version = "1.2.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e46a2036019fdb888131db7a4c847a1063a7493f971ed94ea82c67eada63ca54" +dependencies = [ + "serde", + "serde_derive", +] + [[package]] name = "rustc-demangle" version = "0.1.24" @@ -3505,6 +3682,15 @@ version = "1.3.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "0fda2ff0d084019ba4d7c6f371c95d8fd75ce3524c3cb8fb653a3023f6323e64" +[[package]] +name = "sketches-ddsketch" +version = "0.2.2" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "85636c14b73d81f541e525f585c0a2109e6744e1565b5c1668e31c70c10ed65c" +dependencies = [ + "serde", +] + [[package]] name = "slab" version = "0.4.9" @@ -3654,6 +3840,147 @@ dependencies = [ "libc", ] +[[package]] +name = "tantivy" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "f8d0582f186c0a6d55655d24543f15e43607299425c5ad8352c242b914b31856" +dependencies = [ + "aho-corasick", + "arc-swap", + "base64 0.22.1", + "bitpacking", + "byteorder", + "census", + "crc32fast", + "crossbeam-channel", + "downcast-rs", + "fastdivide", + "fnv", + "fs4", + "htmlescape", + "itertools 0.12.1", + "levenshtein_automata", + "log", + "lru", + "lz4_flex", + "measure_time", + "memmap2", + "num_cpus", + "once_cell", + "oneshot", + "rayon", + "regex", + "rust-stemmers", + "rustc-hash", + "serde", + "serde_json", + "sketches-ddsketch", + "smallvec 1.14.0", + "tantivy-bitpacker", + "tantivy-columnar", + "tantivy-common", + "tantivy-fst", + "tantivy-query-grammar", + "tantivy-stacker", + "tantivy-tokenizer-api", + "tempfile", + "thiserror 1.0.69", + "time", + "uuid", + "winapi", +] + +[[package]] +name = "tantivy-bitpacker" +version = "0.6.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "284899c2325d6832203ac6ff5891b297fc5239c3dc754c5bc1977855b23c10df" +dependencies = [ + "bitpacking", +] + +[[package]] +name = "tantivy-columnar" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "12722224ffbe346c7fec3275c699e508fd0d4710e629e933d5736ec524a1f44e" +dependencies = [ + "downcast-rs", + "fastdivide", + "itertools 0.12.1", + "serde", + "tantivy-bitpacker", + "tantivy-common", + "tantivy-sstable", + "tantivy-stacker", +] + +[[package]] +name = "tantivy-common" +version = "0.7.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8019e3cabcfd20a1380b491e13ff42f57bb38bf97c3d5fa5c07e50816e0621f4" +dependencies = [ + "async-trait", + "byteorder", + "ownedbytes", + "serde", + "time", +] + +[[package]] +name = "tantivy-fst" +version = "0.5.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "d60769b80ad7953d8a7b2c70cdfe722bbcdcac6bccc8ac934c40c034d866fc18" +dependencies = [ + "byteorder", + "regex-syntax", + "utf8-ranges", +] + +[[package]] +name = "tantivy-query-grammar" +version = "0.22.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "847434d4af57b32e309f4ab1b4f1707a6c566656264caa427ff4285c4d9d0b82" +dependencies = [ + "nom", +] + +[[package]] +name = "tantivy-sstable" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c69578242e8e9fc989119f522ba5b49a38ac20f576fc778035b96cc94f41f98e" +dependencies = [ + "tantivy-bitpacker", + "tantivy-common", + "tantivy-fst", + "zstd", +] + +[[package]] +name = "tantivy-stacker" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "c56d6ff5591fc332739b3ce7035b57995a3ce29a93ffd6012660e0949c956ea8" +dependencies = [ + "murmurhash32", + "rand_distr", + "tantivy-common", +] + +[[package]] +name = "tantivy-tokenizer-api" +version = "0.3.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "2a0dcade25819a89cfe6f17d932c9cedff11989936bf6dd4f336d50392053b04" +dependencies = [ + "serde", +] + [[package]] name = "tempfile" version = "3.19.1" @@ -4030,6 +4357,12 @@ version = "1.0.5" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "c8232dd3cdaed5356e0f716d285e4b40b932ac434100fe9b7e0e8e935b9e6246" +[[package]] +name = "utf8-ranges" +version = "1.0.5" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "7fcfc827f90e53a02eaef5e535ee14266c1d569214c6aa70133a624d8a3164ba" + [[package]] name = "utf8_iter" version = "1.0.4" @@ -4042,6 +4375,16 @@ version = "0.2.2" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "06abde3611657adf66d383f00b093d7faecc7fa57071cce2578660c9f1010821" +[[package]] +name = "uuid" +version = "1.16.0" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "458f7a779bf54acc9f347480ac654f68407d3aab21269a6e3c9f922acd9e2da9" +dependencies = [ + "getrandom 0.3.2", + "serde", +] + [[package]] name = "vcpkg" version = "0.2.15" @@ -4654,3 +4997,31 @@ dependencies = [ "quote", "syn 2.0.100", ] + +[[package]] +name = "zstd" +version = "0.13.3" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "e91ee311a569c327171651566e07972200e76fcfe2242a4fa446149a3881c08a" +dependencies = [ + "zstd-safe", +] + +[[package]] +name = "zstd-safe" +version = "7.2.4" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "8f49c4d5f0abb602a93fb8736af2a4f4dd9512e36f7f570d66e65ff867ed3b9d" +dependencies = [ + "zstd-sys", +] + +[[package]] +name = "zstd-sys" +version = "2.0.15+zstd.1.5.7" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "eb81183ddd97d0c74cedf1d50d85c8d08c1b8b68ee863bdee9e706eedba1a237" +dependencies = [ + "cc", + "pkg-config", +] diff --git a/Cargo.toml b/Cargo.toml index adbb260c37..bb4b5a8347 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -100,6 +100,7 @@ tokio-stream = "0.1.9" tokio-util = {version = "0.7.3", features = ["compat"] } tower-http = { version = "0.6.2", features = ["auth", "compression-br", "compression-gzip", "cors", "set-header"] } urlencoding = "2.1.3" +tantivy = "0.22.0" [dev-dependencies] criterion = "0.5.1" diff --git a/src/index.rs b/src/index.rs index 0cb54b3c83..1f169361ff 100644 --- a/src/index.rs +++ b/src/index.rs @@ -1941,6 +1941,24 @@ impl Index { Ok((inscriptions, more)) } + pub fn get_inscriptions_by_sequence_range( + &self, + start_sequence: u32, + end_sequence: u32, + ) -> Result> { + let rtx = self.database.begin_read()?; + + let sequence_number_to_inscription_entry = + rtx.open_table(SEQUENCE_NUMBER_TO_INSCRIPTION_ENTRY)?; + + let inscriptions = sequence_number_to_inscription_entry + .range(start_sequence..=end_sequence)? + .map(|result| result.map(|(_number, entry)| InscriptionEntry::load(entry.value()).id)) + .collect::, StorageError>>()?; + + Ok(inscriptions) + } + pub fn get_inscriptions_in_block(&self, block_height: u32) -> Result> { let rtx = self.database.begin_read()?; diff --git a/src/lib.rs b/src/lib.rs index c12e69d7ce..c3a700deab 100644 --- a/src/lib.rs +++ b/src/lib.rs @@ -30,6 +30,7 @@ use { properties::Properties, representation::Representation, satscard::Satscard, + search_index::SearchIndex, settings::Settings, signer::Signer, subcommand::{OutputFormat, Subcommand, SubcommandResult}, @@ -132,6 +133,7 @@ mod re; mod representation; pub mod runes; mod satscard; +mod search_index; pub mod settings; mod signer; pub mod subcommand; @@ -148,6 +150,7 @@ const TARGET_POSTAGE: Amount = Amount::from_sat(10_000); static SHUTTING_DOWN: AtomicBool = AtomicBool::new(false); static LISTENERS: Mutex> = Mutex::new(Vec::new()); static INDEXER: Mutex>> = Mutex::new(None); +static SEARCH_INDEXER: Mutex>> = Mutex::new(None); #[doc(hidden)] #[derive(Deserialize, Serialize)] @@ -279,6 +282,14 @@ fn gracefully_shut_down_indexer() { log::warn!("Index thread panicked; join failed"); } } + + if let Some(search_indexer) = SEARCH_INDEXER.lock().unwrap().take() { + shut_down(); + log::info!("Waiting for search index thread to finish..."); + if search_indexer.join().is_err() { + log::warn!("Search index thread panicked; join failed"); + } + } } pub fn main() { diff --git a/src/options.rs b/src/options.rs index 5eea5de4db..e26b7bb0d4 100644 --- a/src/options.rs +++ b/src/options.rs @@ -73,6 +73,8 @@ pub struct Options { help = "Do not index inscriptions." )] pub(crate) no_index_inscriptions: bool, + #[arg(long, help = "Use search index at .")] + pub(crate) search_index: Option, #[arg( long, help = "Require basic HTTP authentication with . Credentials are sent in cleartext. Consider using authentication in conjunction with HTTPS." diff --git a/src/search_index.rs b/src/search_index.rs new file mode 100644 index 0000000000..e16b21e19d --- /dev/null +++ b/src/search_index.rs @@ -0,0 +1,206 @@ +use { + super::*, + crate::subcommand::server::query, + tantivy::{ + collector::{Count, TopDocs}, + directory::MmapDirectory, + query::QueryParser, + schema::{ + document::OwnedValue, DateOptions, DateTimePrecision, Field, Schema as TantivySchema, + INDEXED, STORED, STRING, + }, + DateTime, Index as TantivyIndex, IndexReader, IndexWriter, ReloadPolicy, TantivyDocument, + }, +}; + +#[derive(Clone)] +struct Schema { + inscription_id: Field, + charm: Field, + sat_name: Field, + timestamp: Field, +} + +impl Schema { + fn default_search_fields(&self) -> Vec { + vec![ + self.inscription_id, + self.charm, + self.sat_name, + self.timestamp, + ] + } + + fn search_result(&self, document: &TantivyDocument) -> Option { + let inscription_id = document.get_first(self.inscription_id).and_then(|value| { + if let OwnedValue::Str(id_str) = value { + Some(id_str) + } else { + None + } + })?; + + Some(SearchResult { + inscription_id: inscription_id.parse().ok()?, + }) + } +} + +#[derive(Clone)] +pub struct SearchIndex { + ord_index: Arc, + reader: IndexReader, + schema: Schema, + search_index: TantivyIndex, + writer: Arc>, +} + +#[derive(Eq, Hash, PartialEq)] +pub struct SearchResult { + pub inscription_id: InscriptionId, +} + +impl SearchIndex { + pub fn open(index: Arc, settings: &Settings) -> Result { + let mut schema_builder = TantivySchema::builder(); + + let schema = Schema { + inscription_id: schema_builder.add_text_field("inscription_id", STRING | STORED), + charm: schema_builder.add_text_field("charm", STRING), + sat_name: schema_builder.add_text_field("sat_name", STRING), + timestamp: schema_builder.add_date_field( + "timestamp", + DateOptions::from(INDEXED) + .set_fast() + .set_precision(DateTimePrecision::Seconds), + ), + }; + + let path = settings.search_index().to_owned(); + + fs::create_dir_all(&path).snafu_context(error::Io { path: path.clone() })?; + + let search_index = + TantivyIndex::open_or_create(MmapDirectory::open(path)?, schema_builder.build())?; + + let reader = search_index + .reader_builder() + .reload_policy(ReloadPolicy::OnCommitWithDelay) + .try_into()?; + + let writer = search_index.writer(50_000_000)?; + + Ok(Self { + ord_index: index, + reader, + schema, + search_index, + writer: Arc::new(Mutex::new(writer)), + }) + } + + pub fn search(&self, query: &str) -> Result> { + let searcher = self.reader.searcher(); + + Ok( + searcher + .search( + &self.query_parser().parse_query(query)?, + &TopDocs::with_limit(100), + )? + .iter() + .filter_map(|(_score, doc_address)| { + self + .schema + .search_result(&searcher.doc::(*doc_address).ok()?) + }) + .collect(), + ) + } + + pub fn update(&self) -> Result { + let batch_size = 100; + + let mut starting_sequence_number = 0; + + let mut writer = self.writer.lock().unwrap(); + + loop { + let batch = self.ord_index.get_inscriptions_by_sequence_range( + starting_sequence_number, + starting_sequence_number + batch_size, + )?; + + if batch.is_empty() { + return Ok(()); + } + + for inscription_id in batch { + self.add_inscription(inscription_id, &mut writer)?; + + if SHUTTING_DOWN.load(atomic::Ordering::Relaxed) { + writer.commit()?; + return Ok(()); + } + } + + writer.commit()?; + + starting_sequence_number += batch_size; + } + } + + fn add_inscription(&self, inscription_id: InscriptionId, writer: &mut IndexWriter) -> Result { + let searcher = self.reader.searcher(); + + let query = self + .query_parser() + .parse_query(&format!("inscription_id:{inscription_id}"))?; + + if searcher.search(&query, &Count)? > 0 { + return Ok(()); + } + + let (inscription, _, _) = self + .ord_index + .inscription_info(query::Inscription::Id(inscription_id), None)? + .ok_or(anyhow!(format!( + "failed to get info for inscription with id `{inscription_id}`" + )))?; + + let mut document = TantivyDocument::default(); + + document.add_text(self.schema.inscription_id, inscription.id.to_string()); + + for charm in inscription.charms { + document.add_text(self.schema.charm, charm); + } + + if let Some(sat) = inscription.sat { + document.add_text(self.schema.sat_name, sat.name()); + } + + document.add_date( + self.schema.timestamp, + DateTime::from_timestamp_secs(inscription.timestamp), + ); + + writer.add_document(document)?; + + log::info!( + "Added inscription with id `{}` to search index", + inscription_id + ); + + Ok(()) + } + + fn query_parser(&self) -> QueryParser { + let mut query_parser = + QueryParser::for_index(&self.search_index, self.schema.default_search_fields()); + + query_parser.set_conjunction_by_default(); + + query_parser + } +} diff --git a/src/settings.rs b/src/settings.rs index 9f814eac7a..a8b7b77765 100644 --- a/src/settings.rs +++ b/src/settings.rs @@ -10,8 +10,6 @@ pub struct Settings { bitcoin_rpc_username: Option, chain: Option, commit_interval: Option, - savepoint_interval: Option, - max_savepoints: Option, config: Option, config_dir: Option, cookie_file: Option, @@ -26,7 +24,10 @@ pub struct Settings { index_sats: bool, index_transactions: bool, integration_test: bool, + max_savepoints: Option, no_index_inscriptions: bool, + savepoint_interval: Option, + search_index: Option, server_password: Option, server_url: Option, server_username: Option, @@ -118,6 +119,7 @@ impl Settings { chain: self.chain.or(source.chain), commit_interval: self.commit_interval.or(source.commit_interval), savepoint_interval: self.savepoint_interval.or(source.savepoint_interval), + search_index: self.search_index.or(source.search_index), max_savepoints: self.max_savepoints.or(source.max_savepoints), config: self.config.or(source.config), config_dir: self.config_dir.or(source.config_dir), @@ -180,6 +182,7 @@ impl Settings { index_transactions: options.index_transactions, integration_test: options.integration_test, no_index_inscriptions: options.no_index_inscriptions, + search_index: options.search_index, server_password: options.server_password, server_url: None, server_username: options.server_username, @@ -270,6 +273,7 @@ impl Settings { index_transactions: get_bool("INDEX_TRANSACTIONS"), integration_test: get_bool("INTEGRATION_TEST"), no_index_inscriptions: get_bool("NO_INDEX_INSCRIPTIONS"), + search_index: get_path("SEARCH_INDEX"), server_password: get_string("SERVER_PASSWORD"), server_url: get_string("SERVER_URL"), server_username: get_string("SERVER_USERNAME"), @@ -302,6 +306,7 @@ impl Settings { index_transactions: false, integration_test: false, no_index_inscriptions: false, + search_index: None, server_password: None, server_url: Some(server_url.into()), server_username: None, @@ -341,6 +346,11 @@ impl Settings { None => data_dir.join("index.redb"), }; + let search_index = match &self.search_index { + Some(path) => path.clone(), + None => data_dir.join("search-index"), + }; + Ok(Self { bitcoin_data_dir: Some(bitcoin_data_dir), bitcoin_rpc_limit: Some(self.bitcoin_rpc_limit.unwrap_or(12)), @@ -378,6 +388,7 @@ impl Settings { index_transactions: self.index_transactions, integration_test: self.integration_test, no_index_inscriptions: self.no_index_inscriptions, + search_index: Some(search_index), server_password: self.server_password, server_url: self.server_url, server_username: self.server_username, @@ -595,6 +606,10 @@ impl Settings { self.bitcoin_rpc_limit.unwrap() } + pub fn search_index(&self) -> &Path { + self.search_index.as_ref().unwrap() + } + pub fn server_url(&self) -> Option<&str> { self.server_url.as_deref() } @@ -1083,6 +1098,7 @@ mod tests { ("INDEX_TRANSACTIONS", "1"), ("INTEGRATION_TEST", "1"), ("NO_INDEX_INSCRIPTIONS", "1"), + ("SEARCH_INDEX", "search-index"), ("SERVER_PASSWORD", "server password"), ("SERVER_URL", "server url"), ("SERVER_USERNAME", "server username"), @@ -1129,6 +1145,7 @@ mod tests { index_transactions: true, integration_test: true, no_index_inscriptions: true, + search_index: Some("search-index".into()), server_password: Some("server password".into()), server_url: Some("server url".into()), server_username: Some("server username".into()), @@ -1164,6 +1181,7 @@ mod tests { "--index=index", "--integration-test", "--no-index-inscriptions", + "--search-index=search-index", "--server-password=server password", "--server-username=server username", ]) @@ -1194,6 +1212,7 @@ mod tests { index_transactions: true, integration_test: true, no_index_inscriptions: true, + search_index: Some("search-index".into()), server_password: Some("server password".into()), server_url: None, server_username: Some("server username".into()), diff --git a/src/subcommand.rs b/src/subcommand.rs index d3213303a7..f9a5f2aaf8 100644 --- a/src/subcommand.rs +++ b/src/subcommand.rs @@ -72,10 +72,11 @@ impl Subcommand { Self::Parse(parse) => parse.run(), Self::Runes => runes::run(settings), Self::Server(server) => { - let index = Arc::new(Index::open(&settings)?); + let index = Arc::new(Index::open_with_event_sender(&settings, None)?); + let search_index = Arc::new(SearchIndex::open(index.clone(), &settings)?); let handle = axum_server::Handle::new(); LISTENERS.lock().unwrap().push(handle.clone()); - server.run(settings, index, handle) + server.run(settings, index, Some(search_index), handle) } Self::Settings => settings::run(settings), Self::Subsidy(subsidy) => subsidy.run(), diff --git a/src/subcommand/server.rs b/src/subcommand/server.rs index 88084ca39d..d616bc49fe 100644 --- a/src/subcommand/server.rs +++ b/src/subcommand/server.rs @@ -7,12 +7,12 @@ use { }, super::*, crate::templates::{ - AddressHtml, BlockHtml, BlocksHtml, ChildrenHtml, ClockSvg, CollectionsHtml, HomeHtml, - InputHtml, InscriptionHtml, InscriptionsBlockHtml, InscriptionsHtml, OutputHtml, PageContent, - PageHtml, ParentsHtml, PreviewAudioHtml, PreviewCodeHtml, PreviewFontHtml, PreviewIframeHtml, - PreviewImageHtml, PreviewMarkdownHtml, PreviewModelHtml, PreviewPdfHtml, PreviewTextHtml, - PreviewUnknownHtml, PreviewVideoHtml, RareTxt, RuneHtml, RuneNotFoundHtml, RunesHtml, SatHtml, - SatscardHtml, TransactionHtml, + AddressHtml, BlockHtml, BlocksHtml, ChildrenHtml, ClockSvg, CollectionsHtml, ExploreHtml, + HomeHtml, InputHtml, InscriptionHtml, InscriptionsBlockHtml, InscriptionsHtml, OutputHtml, + PageContent, PageHtml, ParentsHtml, PreviewAudioHtml, PreviewCodeHtml, PreviewFontHtml, + PreviewIframeHtml, PreviewImageHtml, PreviewMarkdownHtml, PreviewModelHtml, PreviewPdfHtml, + PreviewTextHtml, PreviewUnknownHtml, PreviewVideoHtml, RareTxt, RuneHtml, RuneNotFoundHtml, + RunesHtml, SatHtml, SatscardHtml, TransactionHtml, }, axum::{ extract::{DefaultBodyLimit, Extension, Json, Path, Query}, @@ -72,6 +72,11 @@ pub(crate) enum OutputType { Runic, } +#[derive(Deserialize)] +struct Explore { + query: Option, +} + #[derive(Deserialize)] struct Search { query: String, @@ -146,7 +151,13 @@ pub struct Server { } impl Server { - pub fn run(self, settings: Settings, index: Arc, handle: Handle) -> SubcommandResult { + pub fn run( + self, + settings: Settings, + index: Arc, + search_index: Option>, + handle: Handle, + ) -> SubcommandResult { Runtime::new()?.block_on(async { let index_clone = index.clone(); let integration_test = settings.integration_test(); @@ -171,6 +182,30 @@ impl Server { INDEXER.lock().unwrap().replace(index_thread); + if let Some(search_index) = &search_index { + let search_index_clone = search_index.clone(); + + let search_index_thread = thread::spawn(move || loop { + if SHUTTING_DOWN.load(atomic::Ordering::Relaxed) { + break; + } + + if !self.no_sync { + if let Err(error) = search_index_clone.update() { + log::warn!("Updating search index: {error}"); + } + } + + thread::sleep(if integration_test { + Duration::from_millis(100) + } else { + self.polling_interval.into() + }); + }); + + SEARCH_INDEXER.lock().unwrap().replace(search_index_thread); + } + let settings = Arc::new(settings); let acme_domains = self.acme_domains()?; @@ -185,7 +220,7 @@ impl Server { }); // non-recursive endpoints - let router = Router::new() + let mut router = Router::new() .route("/", get(Self::home)) .route("/address/{address}", get(Self::address)) .route("/block/{query}", get(Self::block)) @@ -247,6 +282,15 @@ impl Server { .route("/tx/{txid}", get(Self::transaction)) .route("/update", get(Self::update)); + if let Some(search_index) = search_index { + router = router.nest( + "/explore", + Router::new() + .route("/", get(Self::explore)) + .layer(Extension(search_index)), + ); + } + // recursive endpoints let router = router .route("/blockhash", get(r::blockhash_string)) @@ -591,6 +635,23 @@ impl Server { }) } + async fn explore( + Extension(server_config): Extension>, + Extension(search_index): Extension>, + Query(explore): Query, + ) -> ServerResult { + let search_results = match explore.query { + Some(query) => search_index.search(&query)?, + None => Vec::new(), + }; + + Ok( + ExploreHtml { search_results } + .page(server_config) + .into_response(), + ) + } + async fn fallback(Extension(index): Extension>, uri: Uri) -> ServerResult { task::block_in_place(|| { let path = urlencoding::decode(uri.path().trim_matches('/')) @@ -2121,7 +2182,11 @@ mod tests { { let index = index.clone(); let ord_server_handle = ord_server_handle.clone(); - thread::spawn(|| server.run(settings, index, ord_server_handle).unwrap()); + thread::spawn(|| { + server + .run(settings, index, None, ord_server_handle) + .unwrap() + }); } while index.statistic(crate::index::Statistic::Commits) == 0 { diff --git a/src/templates.rs b/src/templates.rs index dea29c786a..9fe0373879 100644 --- a/src/templates.rs +++ b/src/templates.rs @@ -7,6 +7,7 @@ pub(crate) use { children::ChildrenHtml, clock::ClockSvg, collections::CollectionsHtml, + explore::ExploreHtml, home::HomeHtml, iframe::Iframe, input::InputHtml, @@ -37,6 +38,7 @@ pub mod blocks; mod children; mod clock; pub mod collections; +mod explore; mod home; mod iframe; mod input; diff --git a/src/templates/explore.rs b/src/templates/explore.rs new file mode 100644 index 0000000000..b52df016db --- /dev/null +++ b/src/templates/explore.rs @@ -0,0 +1,12 @@ +use {super::*, search_index::SearchResult}; + +#[derive(Boilerplate)] +pub(crate) struct ExploreHtml { + pub(crate) search_results: Vec, +} + +impl PageContent for ExploreHtml { + fn title(&self) -> String { + "Explore".to_string() + } +} diff --git a/templates/explore.html b/templates/explore.html new file mode 100644 index 0000000000..41da73108c --- /dev/null +++ b/templates/explore.html @@ -0,0 +1,15 @@ +

Explore

+ +
+ + +
+ +%% if !self.search_results.is_empty() { +

Results ({{ self.search_results.len() }})

+
+%% for result in &self.search_results { + {{ Iframe::thumbnail(result.inscription_id) }} +%% } +
+%% } diff --git a/tests/settings.rs b/tests/settings.rs index 0280e16df8..4bf4fb0457 100644 --- a/tests/settings.rs +++ b/tests/settings.rs @@ -13,8 +13,6 @@ fn default() { "bitcoin_rpc_username": null, "chain": "mainnet", "commit_interval": 5000, - "savepoint_interval": 10, - "max_savepoints": 2, "config": null, "config_dir": null, "cookie_file": ".*\.cookie", @@ -29,7 +27,10 @@ fn default() { "index_sats": false, "index_transactions": false, "integration_test": false, + "max_savepoints": 2, "no_index_inscriptions": false, + "savepoint_interval": 10, + "search_index": ".*search-index", "server_password": null, "server_url": null, "server_username": null diff --git a/tests/test_server.rs b/tests/test_server.rs index 57bdce2a8b..1cf5ddc4e3 100644 --- a/tests/test_server.rs +++ b/tests/test_server.rs @@ -57,7 +57,11 @@ impl TestServer { { let index = index.clone(); let ord_server_handle = ord_server_handle.clone(); - thread::spawn(|| server.run(settings, index, ord_server_handle).unwrap()); + thread::spawn(|| { + server + .run(settings, index, None, ord_server_handle) + .unwrap() + }); } for i in 0.. {