makefile extracts tokenizer version from dockerfile

Signed-off-by: Maroon Ayoub <[email protected]>

Author: vMaroon

Dockerfile

@@ -39,7 +39,8 @@ COPY . .
 
 # HuggingFace tokenizer bindings
 RUN mkdir -p lib
-RUN curl -L https://github.com/daulet/tokenizers/releases/download/v1.22.1/libtokenizers.${TARGETOS}-${TARGETARCH}.tar.gz | tar -xz -C lib
+ARG RELEASE_VERSION=1.22.1
+RUN curl -L https://github.com/daulet/tokenizers/releases/download/${RELEASE_VERSION}/libtokenizers.${TARGETOS}-${TARGETARCH}.tar.gz | tar -xz -C lib
 RUN ranlib lib/*.a
 
 # Build

Makefile

@@ -27,15 +27,16 @@ help: ## Print help
 LDFLAGS ?= -extldflags '-L$(shell pwd)/lib'
 CGO_ENABLED=1
 TOKENIZER_LIB = lib/libtokenizers.a
-TOKENIZER_RELEASE = v1.22.1
+# Extract RELEASE_VERSION from Dockerfile
+TOKENIZER_VERSION := $(shell grep '^ARG RELEASE_VERSION=' Dockerfile | cut -d'=' -f2)
 
 .PHONY: download-tokenizer
 download-tokenizer: $(TOKENIZER_LIB)
 $(TOKENIZER_LIB):
 	## Download the HuggingFace tokenizer bindings.
-	@echo "Downloading HuggingFace tokenizer bindings..."
+	@echo "Downloading HuggingFace tokenizer bindings for version v$(TOKENIZER_VERSION)..."
 	mkdir -p lib
-	curl -L https://github.com/daulet/tokenizers/releases/download/$(TOKENIZER_RELEASE)/libtokenizers.$(TARGETOS)-$(TARGETARCH).tar.gz | tar -xz -C lib
+	curl -L https://github.com/daulet/tokenizers/releases/download/v$(TOKENIZER_VERSION)/libtokenizers.$(TARGETOS)-$(TARGETARCH).tar.gz | tar -xz -C lib
 	ranlib lib/*.a
 
 ##@ Precommit code checks --

examples/kv_cache_index/main.go

@@ -94,6 +94,8 @@ func setupKVCacheIndexer(ctx context.Context) (*kvcache.Indexer, error) {
 		return nil, err
 	}
 
+	config.TokenProcessorConfig.BlockSize = 256
+
 	kvCacheIndexer, err := kvcache.NewKVCacheIndexer(ctx, config)
 	if err != nil {
 		return nil, err

examples/kv_events/offline/main.go

@@ -46,6 +46,8 @@ func getKVCacheIndexerConfig() *kvcache.Config {
 		config.TokenizersPoolConfig.HuggingFaceToken = huggingFaceToken
 	}
 
+	config.TokenProcessorConfig.BlockSize = 256
+
 	return config
 }