Linter

.github/workflows/linter.yml

@@ -0,0 +1,27 @@
+name: Ruff
+
+on:
+  push:
+    branches: [ "main" ]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [ "main" ]
+  schedule:
+    - cron: '30 3 * * *'
+
+permissions:
+  contents: read
+
+jobs:
+  ruff:
+    name: Ruff
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v4
+      - uses: actions/setup-python@v5
+        with:
+          python-version: '3.12'
+          cache: 'pip'
+      - uses: chartboost/ruff-action@v1
+        with:
+          args: 'check --select B,F,I,PERF,NPY,PL,RUF,S,SIM,UP --output-format github --diff --exclude docker,docs,scripts,tests,vendor'

examples/batch-processing/server.py

@@ -23,8 +23,6 @@
 
 app = FastAPI()
 
-import openai.types.chat as types
-
 @app.post("/v1/chat/completions")
 def create_chat_completions():
     return {"message": "Hello World"}

examples/gradio_chat/local.py

@@ -1,8 +1,8 @@
+import gradio as gr
+
 import llama_cpp
 import llama_cpp.llama_tokenizer
 
-import gradio as gr
-
 llama = llama_cpp.Llama.from_pretrained(
     repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
     filename="*q8_0.gguf",

examples/gradio_chat/server.py

@@ -1,5 +1,4 @@
 import gradio as gr
-
 from openai import OpenAI
 
 client = OpenAI(

examples/hf_pull/main.py

@@ -1,7 +1,6 @@
 import llama_cpp
 import llama_cpp.llama_tokenizer
 
-
 llama = llama_cpp.Llama.from_pretrained(
     repo_id="Qwen/Qwen1.5-0.5B-Chat-GGUF",
     filename="*q8_0.gguf",
@@ -36,4 +35,4 @@
         continue
     print(delta["content"], end="", flush=True)
 
-print()
+print()

examples/high_level_api/fastapi_server.py

@@ -25,6 +25,7 @@
 
 """
 import os
+
 import uvicorn
 
 from llama_cpp.server.app import create_app

examples/high_level_api/high_level_api_inference.py

@@ -1,5 +1,5 @@
-import json
 import argparse
+import json
 
 from llama_cpp import Llama
 

examples/high_level_api/high_level_api_streaming.py

@@ -1,5 +1,5 @@
-import json
 import argparse
+import json
 
 from llama_cpp import Llama
 

examples/high_level_api/langchain_custom_llm.py

@@ -1,9 +1,9 @@
 import argparse
-
-from llama_cpp import Llama
+from typing import Any, List, Mapping, Optional
 
 from langchain.llms.base import LLM
-from typing import Optional, List, Mapping, Any
+
+from llama_cpp import Llama
 
 
 class LlamaLLM(LLM):
@@ -42,8 +42,8 @@ def _identifying_params(self) -> Mapping[str, Any]:
 print(f"Answer: {answer.strip()}")
 
 # Using in a chain
-from langchain.prompts import PromptTemplate
 from langchain.chains import LLMChain
+from langchain.prompts import PromptTemplate
 
 prompt = PromptTemplate(
     input_variables=["product"],

examples/low_level_api/Chat.py

@@ -1,8 +1,12 @@
 #!/bin/python
-import sys, os, datetime
+import datetime
+import os
+import sys
+
 from common import GptParams
 from low_level_api_chat_cpp import LLaMAInteract
 
+
 def env_or_def(env, default):
 	if (env in os.environ):
 		return os.environ[env]

examples/low_level_api/Miku.py

@@ -1,8 +1,11 @@
 #!/bin/python
-import sys, os
+import os
+import sys
+
 from common import GptParams
 from low_level_api_chat_cpp import LLaMAInteract
 
+
 def env_or_def(env, default):
 	if (env in os.environ):
 		return os.environ[env]

examples/low_level_api/ReasonAct.py

@@ -1,16 +1,19 @@
 #!/bin/python
-import sys, os, datetime
+import os
+import sys
+
 from common import GptParams
 from low_level_api_chat_cpp import LLaMAInteract
 
+
 def env_or_def(env, default):
 	if (env in os.environ):
 		return os.environ[env]
 	return default
 
 MODEL = env_or_def("MODEL", "./models/llama-13B/ggml-model.bin")
 
-prompt=f"""You run in a loop of Thought, Action, Observation.
+prompt="""You run in a loop of Thought, Action, Observation.
 At the end of the loop either Answer or restate your Thought and Action.
 Use Thought to describe your thoughts about the question you have been asked.
 Use Action to run one of these actions available to you:

examples/low_level_api/common.py

@@ -1,7 +1,6 @@
-import os
 import argparse
+import os
 import re
-
 from dataclasses import dataclass, field
 from typing import List
 

examples/low_level_api/low_level_api_chat_cpp.py

@@ -12,12 +12,14 @@
 """
 import ctypes
 import sys
-from time import time
 from os import cpu_count, path
+from time import time
 
-import llama_cpp
-from common import GptParams, gpt_params_parse, gpt_random_prompt
 import util
+from common import GptParams, gpt_params_parse, gpt_random_prompt
+
+import llama_cpp
+
 
 # A LLaMA interactive session
 class LLaMAInteract:
@@ -130,13 +132,13 @@ def __init__(self, params: GptParams) -> None:
 					self.params.n_ctx,
 					ctypes.byref(_n_token_count_out)
 				) != 1):
-					print(f"error: failed to load session file '{self.params.path_session}'", file=sys.stderr)
+					print("error: failed to load session file", file=sys.stderr)
 					return
 				_n_token_count_out = _n_token_count_out.value
 				self.session_tokens = _session_tokens[:_n_token_count_out]
 				print(f"loaded a session with prompt size of {_n_token_count_out} tokens", file=sys.stderr)
 			else:
-				print(f"session file does not exist, will create", file=sys.stderr)
+				print("session file does not exist, will create", file=sys.stderr)
 
 		# tokenize the prompt
 		self.embd = []
@@ -154,7 +156,7 @@ def __init__(self, params: GptParams) -> None:
 				self.n_matching_session_tokens += 1
 
 			if self.n_matching_session_tokens >= len(self.embd_inp):
-				print(f"session file has exact match for prompt!")
+				print("session file has exact match for prompt!")
 			elif self.n_matching_session_tokens < (len(self.embd_inp) / 2):
 				print(f"warning: session file has low similarity to prompt ({self.n_matching_session_tokens} / {len(self.embd_inp)} tokens); will mostly be reevaluated")
 			else:
@@ -374,24 +376,23 @@ def generate(self):
 				if self.params.temp <= 0:
 					# Greedy sampling
 					id = llama_cpp.llama_sample_token_greedy(self.ctx, candidates_p)
+				elif self.params.mirostat == 1:
+					mirostat_mu = 2.0 * self.params.mirostat_tau
+					mirostat_m = 100
+					llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp))
+					id = llama_cpp.llama_sample_token_mirostat(self.ctx, candidates_p, llama_cpp.c_float(self.params.mirostat_tau), llama_cpp.c_float(self.params.mirostat_eta), llama_cpp.c_int(mirostat_m), llama_cpp.c_float(mirostat_mu))
+				elif self.params.mirostat == 2:
+					mirostat_mu = 2.0 * self.params.mirostat_tau
+					llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp))
+					id = llama_cpp.llama_sample_token_mirostat_v2(self.ctx, candidates_p, llama_cpp.c_float(self.params.mirostat_tau), llama_cpp.c_float(self.params.mirostat_eta), llama_cpp.c_float(mirostat_mu))
 				else:
-					if self.params.mirostat == 1:
-						mirostat_mu = 2.0 * self.params.mirostat_tau
-						mirostat_m = 100
-						llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp))
-						id = llama_cpp.llama_sample_token_mirostat(self.ctx, candidates_p, llama_cpp.c_float(self.params.mirostat_tau), llama_cpp.c_float(self.params.mirostat_eta), llama_cpp.c_int(mirostat_m), llama_cpp.c_float(mirostat_mu))
-					elif self.params.mirostat == 2:
-						mirostat_mu = 2.0 * self.params.mirostat_tau
-						llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp))
-						id = llama_cpp.llama_sample_token_mirostat_v2(self.ctx, candidates_p, llama_cpp.c_float(self.params.mirostat_tau), llama_cpp.c_float(self.params.mirostat_eta), llama_cpp.c_float(mirostat_mu))
-					else:
-						# Temperature sampling
-						llama_cpp.llama_sample_top_k(self.ctx, candidates_p, top_k, min_keep=llama_cpp.c_size_t(1))
-						llama_cpp.llama_sample_tail_free(self.ctx, candidates_p, llama_cpp.c_float(self.params.tfs_z), min_keep=llama_cpp.c_size_t(1))
-						llama_cpp.llama_sample_typical(self.ctx, candidates_p, llama_cpp.c_float(self.params.typical_p), min_keep=llama_cpp.c_size_t(1))
-						llama_cpp.llama_sample_top_p(self.ctx, candidates_p, llama_cpp.c_float(self.params.top_p), min_keep=llama_cpp.c_size_t(1))
-						llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp))
-						id = llama_cpp.llama_sample_token(self.ctx, candidates_p)
+					# Temperature sampling
+					llama_cpp.llama_sample_top_k(self.ctx, candidates_p, top_k, min_keep=llama_cpp.c_size_t(1))
+					llama_cpp.llama_sample_tail_free(self.ctx, candidates_p, llama_cpp.c_float(self.params.tfs_z), min_keep=llama_cpp.c_size_t(1))
+					llama_cpp.llama_sample_typical(self.ctx, candidates_p, llama_cpp.c_float(self.params.typical_p), min_keep=llama_cpp.c_size_t(1))
+					llama_cpp.llama_sample_top_p(self.ctx, candidates_p, llama_cpp.c_float(self.params.top_p), min_keep=llama_cpp.c_size_t(1))
+					llama_cpp.llama_sample_temperature(self.ctx, candidates_p, llama_cpp.c_float(self.params.temp))
+					id = llama_cpp.llama_sample_token(self.ctx, candidates_p)
 				# print("`{}`".format(candidates_p.size))
 
 				self.last_n_tokens.pop(0)

examples/low_level_api/low_level_api_llama_cpp.py

@@ -1,6 +1,6 @@
 import ctypes
-import os
 import multiprocessing
+import os
 
 import llama_cpp
 

examples/low_level_api/quantize.py

@@ -1,5 +1,6 @@
-import os
 import argparse
+import os
+
 import llama_cpp
 
 

examples/low_level_api/util.py

@@ -44,7 +44,7 @@ def append(self, elem):
 
 	def __getitem__(self, val):
 		if isinstance(val, int):
-			if 0 > val or val >= self.size:
+			if val < 0 or val >= self.size:
 				raise IndexError('Index out of range')
 			return self.list[val] if self.size < self.maxsize else self.list[(self.offset + val) % self.maxsize]
 		elif isinstance(val, slice):

examples/ray/llm.py

@@ -1,9 +1,12 @@
-from starlette.requests import Request
 from typing import Dict
+
 from ray import serve
 from ray.serve import Application
+from starlette.requests import Request
+
 from llama_cpp import Llama
 
+
 @serve.deployment
 class LlamaDeployment:
     def __init__(self, model_path: str):

llama_cpp/__init__.py

@@ -1,4 +1,4 @@
-from .llama_cpp import *
 from .llama import *
+from .llama_cpp import *
 
-__version__ = "0.2.79"
+__version__ = "0.2.79"