diff --git a/lib/ruby_indexer/lib/ruby_indexer/configuration.rb b/lib/ruby_indexer/lib/ruby_indexer/configuration.rb
index 25d53029c..f8db6a9d8 100644
--- a/lib/ruby_indexer/lib/ruby_indexer/configuration.rb
+++ b/lib/ruby_indexer/lib/ruby_indexer/configuration.rb
@@ -171,7 +171,7 @@ def apply_config(config)
     sig { returns(String) }
     def merge_exclude_patterns
       relative_patterns = @excluded_patterns
-        .select { |p| p.end_with?("/**/*") }
+        .select { |p| p.end_with?("/**/*") && (p.start_with?(Dir.pwd) || p.start_with?("**")) }
         .map { |p| p.delete_prefix("#{Dir.pwd}/") }
 
       "#{Dir.pwd}/{#{relative_patterns.join(",")}}"