Skip to content

Commit

Permalink
F/squash test again (#3)
Browse files Browse the repository at this point in the history
* add

* small change
  • Loading branch information
lyon-tonic authored Mar 4, 2024
1 parent 5d43de5 commit b2c737a
Showing 1 changed file with 14 additions and 15 deletions.
29 changes: 14 additions & 15 deletions src/runner.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,30 +4,26 @@
from tonic_validate import BenchmarkItem, LLMResponse, ValidateApi, ValidateScorer

path_to_responses = os.environ.get('VALIDATE_RESPONSES_PATH', None)
validate_api_key = os.environ.get('TONIC_VALIDATE_SERVER_API_KEY', None)
validate_project_id = os.environ.get('TONIC_VALIDATE_SERVER_PROJECT_ID', None)
if path_to_responses is None:
exit('Error: You must specify VALIDATE_RESPONSES_PATH, the path to your LLM question and responses')
if not os.path.exists(path_to_responses):
exit('ERROR: The VALIDATE_RESPONSES_PATH provided ("{}") does not exist'.format(path_to_responses))

validate_api_key = os.environ.get('TONIC_VALIDATE_SERVER_API_KEY', None)
if validate_api_key is None:
exit('Error: You must specify TONIC_VALIDATE_SERVER_API_KEY, the API key for the Tonic Validate server')

validate_project_id = os.environ.get('TONIC_VALIDATE_SERVER_PROJECT_ID', None)
if validate_project_id is None:
exit('Error: You must specify TONIC_VALIDATE_SERVER_PROJECT_ID, the project ID for the Tonic Validate server')

if path_to_responses is None:
exit('Error: You must specify VALIDATE_RESPONSES_PATH, the path to your LLM question and responses')

if not os.path.exists(path_to_responses):
exit('ERROR: The VALIDATE_RESPONSES_PATH provided ("{}") does not exist'.format(path_to_responses))

openai_key = os.environ.get("OPENAI_API_KEY")
azure_key = os.environ.get("AZURE_OPENAI_KEY")
azure_endpoint = os.environ.get("AZURE_OPENAI_ENDPOINT")

if openai_key is None or openai_key=='':
if (azure_key is None or azure_key=='') and (azure_endpoint is None or azure_endpoint==''):
exit('ERROR: You must set either an OpenAI key or an Azure key and Azure endpoint')


with open(path_to_responses) as json_data:
try:
responses = json.load(json_data)
Expand All @@ -46,14 +42,17 @@

scorer = ValidateScorer()
run = scorer.score_responses(llm_responses)
validate_api = ValidateApi(validate_api_key)

github_server = os.environ.get('GITHUB_SERVER_URL')
github_repository = os.environ.get('GITHUB_REPOSITORY')
github_sha = os.environ.get('GITHUB_SHA')
# Use default github action environment variables
github_server = os.environ.get('GITHUB_SERVER_URL', None)
github_repository = os.environ.get('GITHUB_REPOSITORY', None)
github_sha = os.environ.get('GITHUB_SHA', None)

run_metadata = {
'commit_url': f"{github_server}/{github_repository}/commit/{github_sha}"
}

validate_api = ValidateApi(validate_api_key)
validate_api.upload_run(validate_project_id, run, run_metadata)
validate_api.upload_run(validate_project_id, run, run_metadata)

print('Run uploaded to Tonic Validate server')

0 comments on commit b2c737a

Please sign in to comment.