Replace full-flush Route cache with LRU and add a configurable max cache size (#724)

Author: sdogruyol

spec/route_handler_spec.cr

@@ -179,4 +179,114 @@ describe "Kemal::RouteHandler" do
     client_response.body.should eq("home page")
     client_response.headers.has_key?("Location").should eq(true)
   end
+
+  context "LRU cache" do
+    it "evicts least recently used entries instead of clearing entirely" do
+      # Use a small capacity to make the test fast and deterministic
+      small_capacity = 8
+      # Replace the cache instance with a smaller-capacity LRU for this test
+      Kemal::RouteHandler::INSTANCE.cached_routes = Kemal::LRUCache(String, Radix::Result(Kemal::Route)).new(small_capacity)
+
+      # Define more routes than capacity
+      0.upto(15) do |i|
+        get "/lru_eviction_#{i}" do
+          "ok"
+        end
+      end
+
+      # Access the first `small_capacity` routes to fill the cache
+      0.upto(small_capacity - 1) do |i|
+        Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_eviction_#{i}")
+      end
+
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq small_capacity
+
+      # Access some new routes to trigger eviction
+      small_capacity.upto(small_capacity + 3) do |i|
+        Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_eviction_#{i}")
+      end
+
+      # Cache should still be capped at capacity
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq small_capacity
+    end
+
+    it "retains recently used keys and evicts the least recently used" do
+      small_capacity = 4
+      Kemal::RouteHandler::INSTANCE.cached_routes = Kemal::LRUCache(String, Radix::Result(Kemal::Route)).new(small_capacity)
+
+      0.upto(5) do |i|
+        get "/lru_recency_#{i}" do
+          "ok"
+        end
+      end
+
+      # Fill cache with 0..3
+      0.upto(3) do |i|
+        Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_recency_#{i}")
+      end
+
+      # Touch 0 and 1 to make them most recent
+      [0, 1].each do |i|
+        Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_recency_#{i}")
+      end
+
+      # Insert 4 -> should evict least recent among {2,3}
+      Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_recency_4")
+
+      # Insert 5 -> should evict the other of {2,3}
+      Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_recency_5")
+
+      # Now 0 and 1 must still resolve from cache, and size is capped
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq small_capacity
+
+      # A fresh lookup for 0 and 1 should be cache hits and not raise
+      Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_recency_0").found?.should be_true
+      Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_recency_1").found?.should be_true
+    end
+
+    it "caches HEAD fallback GET lookups without growing beyond 1 for same path" do
+      cap = 16
+      Kemal::RouteHandler::INSTANCE.cached_routes = Kemal::LRUCache(String, Radix::Result(Kemal::Route)).new(cap)
+
+      get "/head_fallback" do
+        "ok"
+      end
+
+      # First HEAD should fallback to GET and cache one entry keyed by HEAD+path
+      Kemal::RouteHandler::INSTANCE.lookup_route("HEAD", "/head_fallback").found?.should be_true
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq 1
+
+      # Second HEAD lookup should be a cache hit; size must remain 1
+      Kemal::RouteHandler::INSTANCE.lookup_route("HEAD", "/head_fallback").found?.should be_true
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq 1
+    end
+
+    it "keeps size capped under heavy churn with large capacity" do
+      large_capacity = 4096
+      Kemal::RouteHandler::INSTANCE.cached_routes = Kemal::LRUCache(String, Radix::Result(Kemal::Route)).new(large_capacity)
+
+      0.upto(12000) do |i|
+        get "/lru_heavy_#{i}" do
+          "ok"
+        end
+      end
+
+      # Fill and churn beyond capacity
+      0.upto(11999) do |i|
+        Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_heavy_#{i}")
+      end
+
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq large_capacity
+
+      # Additional churn should not increase size
+      12000.upto(14000) do |i|
+        get "/lru_heavy_more_#{i}" do
+          "ok"
+        end
+        Kemal::RouteHandler::INSTANCE.lookup_route("GET", "/lru_heavy_more_#{i}")
+      end
+
+      Kemal::RouteHandler::INSTANCE.cached_routes.size.should eq large_capacity
+    end
+  end
 end

spec/spec_helper.cr

@@ -93,6 +93,6 @@ Spec.after_each do
   Kemal.config.clear
   Kemal::FilterHandler::INSTANCE.tree = Radix::Tree(Array(Kemal::FilterHandler::FilterBlock)).new
   Kemal::RouteHandler::INSTANCE.routes = Radix::Tree(Route).new
-  Kemal::RouteHandler::INSTANCE.cached_routes = Hash(String, Radix::Result(Route)).new
+  Kemal::RouteHandler::INSTANCE.cached_routes = Kemal::LRUCache(String, Radix::Result(Kemal::Route)).new(Kemal.config.max_route_cache_size)
   Kemal::WebSocketHandler::INSTANCE.routes = Radix::Tree(WebSocket).new
 end

src/kemal/config.cr

@@ -26,6 +26,7 @@ module Kemal
     property serve_static : (Bool | Hash(String, Bool))
     property static_headers : (HTTP::Server::Context, String, File::Info -> Void)?
     property? powered_by_header : Bool = true
+    property max_route_cache_size : Int32
 
     def initialize
       @app_name = "Kemal"
@@ -43,6 +44,7 @@ module Kemal
       @running = false
       @shutdown_message = true
       @handler_position = 0
+      @max_route_cache_size = 1024
     end
 
     @[Deprecated("Use standard library Log")]
@@ -69,6 +71,7 @@ module Kemal
       @router_included = false
       @handler_position = 0
       @default_handlers_setup = false
+      @max_route_cache_size = 1024
       HANDLERS.clear
       CUSTOM_HANDLERS.clear
       FILTER_HANDLERS.clear

src/kemal/route_handler.cr

@@ -1,16 +1,114 @@
 require "radix"
 
 module Kemal
+  # Small, private LRU cache used by the router to avoid full cache clears
+  # when many distinct paths are accessed. Keeps get/put at O(1).
+  # This is intentionally minimal and file-local to avoid API surface.
+  class LRUCache(K, V)
+    # Doubly-linked list node
+    class Node(K, V)
+      property key : K
+      property value : V
+      property prev : Node(K, V)?
+      property next : Node(K, V)?
+
+      def initialize(@key : K, @value : V)
+        @prev = nil
+        @next = nil
+      end
+    end
+
+    @capacity : Int32
+    @map : Hash(K, Node(K, V))
+    @head : Node(K, V)? # most-recent
+    @tail : Node(K, V)? # least-recent
+
+    def initialize(@capacity : Int32)
+      @map = Hash(K, Node(K, V)).new
+      @head = nil
+      @tail = nil
+    end
+
+    def size : Int32
+      @map.size
+    end
+
+    def get(key : K) : V?
+      if node = @map[key]?
+        move_to_front(node)
+        return node.value
+      end
+      nil
+    end
+
+    def put(key : K, value : V) : Nil
+      if node = @map[key]?
+        node.value = value
+        move_to_front(node)
+        return
+      end
+
+      # Evict before adding to avoid unnecessary hash resize
+      evict_if_at_capacity
+
+      node = Node(K, V).new(key, value)
+      @map[key] = node
+      insert_front(node)
+    end
+
+    private def insert_front(node : Node(K, V))
+      node.prev = nil
+      node.next = @head
+      @head.try(&.prev=(node))
+      @head = node
+      @tail = node if @tail.nil?
+    end
+
+    private def move_to_front(node : Node(K, V))
+      return if node == @head
+
+      # unlink
+      prev = node.prev
+      nxt = node.next
+      prev.try(&.next=(nxt))
+      nxt.try(&.prev=(prev))
+
+      # fix tail if needed
+      if node == @tail
+        @tail = prev
+      end
+
+      insert_front(node)
+    end
+
+    private def evict_if_at_capacity
+      return if @map.size < @capacity
+
+      if lru = @tail
+        # unlink tail
+        prev = lru.prev
+        if prev
+          prev.next = nil
+          @tail = prev
+        else
+          # only one element
+          @head = nil
+          @tail = nil
+        end
+        @map.delete(lru.key)
+      end
+    end
+  end
+
   class RouteHandler
     include HTTP::Handler
 
-    INSTANCE            = new
-    CACHED_ROUTES_LIMIT = 1024
+    INSTANCE = new
     property routes, cached_routes
 
     def initialize
       @routes = Radix::Tree(Route).new
-      @cached_routes = Hash(String, Radix::Result(Route)).new
+      @cached_routes = LRUCache(String, Radix::Result(Route)).new(Kemal.config.max_route_cache_size)
     end
 
     def call(context : HTTP::Server::Context)
@@ -26,7 +124,7 @@ module Kemal
     def lookup_route(verb : String, path : String)
       lookup_path = radix_path(verb, path)
 
-      if cached_route = @cached_routes[lookup_path]?
+      if cached_route = @cached_routes.get(lookup_path)
         return cached_route
       end
 
@@ -38,8 +136,7 @@ module Kemal
       end
 
       if route.found?
-        @cached_routes.clear if @cached_routes.size == CACHED_ROUTES_LIMIT
-        @cached_routes[lookup_path] = route
+        @cached_routes.put(lookup_path, route)
       end
 
       route