Llm timeout and retry; verbose logging; max_concurrent config option

README.md

@@ -98,7 +98,7 @@ compiler:
   target_dir: terraform
 ```
 
-Set `source_dir` to the directory where your Salami files are, and `target_dir` to the directory where you want the Terraform files to be written. The config file supports environment variables, which is useful to avoid storing secrets in version control. To inject an env variable at runtime, use the `${ENV_VAR}` delimeter.
+Set `compiler.source_dir` to the directory where your Salami files are, and `compiler.target_dir` to the directory where the Terraform files should be written. The config file supports environment variables, which is useful to avoid storing secrets in version control. To inject an env variable at runtime, use the `${ENV_VAR}` delimeter. Use `compiler.llm.max_concurrent` config to control how many concurrent API calls are made to OpenAI API. The default is 5.
 
 ## ✅ VS Code Extension
 

backend/llm/openai/gpt4/client.go

@@ -5,10 +5,12 @@ import (
 	"encoding/json"
 	"math"
 	backendTypes "salami/backend/types"
+	"salami/common/constants"
 	"salami/common/errors"
 	"salami/common/logger"
 	commonTypes "salami/common/types"
 	"strings"
+	"time"
 
 	"github.com/sashabaranov/go-openai"
 )
@@ -21,9 +23,10 @@ const LlmMessageAssistantRole = "assistant"
 const functionCallName = "save_code"
 
 type OpenaiGpt4 struct {
-	slug   string
-	model  string
-	client *openai.Client
+	slug                    string
+	model                   string
+	client                  *openai.Client
+	maxConcurrentExecutions int
 }
 
 type LlmMessage struct {
@@ -35,9 +38,10 @@ type LlmMessage struct {
 
 func NewLlm(llmConfig commonTypes.LlmConfig) backendTypes.Llm {
 	return &OpenaiGpt4{
-		client: getClient(llmConfig),
-		model:  openai.GPT4,
-		slug:   commonTypes.LlmOpenaiGpt4,
+		client:                  getClient(llmConfig),
+		model:                   openai.GPT4,
+		slug:                    commonTypes.LlmOpenaiGpt4,
+		maxConcurrentExecutions: llmConfig.MaxConcurrentExecutions,
 	}
 }
 
@@ -66,7 +70,7 @@ func (o *OpenaiGpt4) GetSlug() string {
 }
 
 func (o *OpenaiGpt4) GetMaxConcurrentExecutions() int {
-	return 5
+	return o.maxConcurrentExecutions
 }
 
 func (o *OpenaiGpt4) CreateCompletion(messages []interface{}) (string, error) {
@@ -78,29 +82,53 @@ func (o *OpenaiGpt4) CreateCompletion(messages []interface{}) (string, error) {
 
 	logMessages(llmMessages)
 
-	response, err := o.client.CreateChatCompletion(
-		context.Background(),
-		o.getChatCompletionRequest(llmMessages),
-	)
+	response, err := o.callApi(llmMessages)
 	if err != nil {
 		return "", err
 	}
+
 	functionCall := response.Choices[0].Message.FunctionCall
 	if functionCall == nil {
 		return "", &errors.LlmError{Message: "Function call is nil"}
 	}
+
 	var parsedArguments map[string]interface{}
 	err = json.Unmarshal([]byte(functionCall.Arguments), &parsedArguments)
 	if err != nil {
 		return "", err
 	}
+
 	code, ok := parsedArguments["code"].(string)
 	if !ok {
 		return "", &errors.LlmError{Message: "Code is not a string"}
 	}
+
 	return strings.TrimSpace(code), nil
 }
 
+func (o *OpenaiGpt4) callApi(llmMessages []*LlmMessage) (openai.ChatCompletionResponse, error) {
+	var response openai.ChatCompletionResponse
+	var err error
+
+	for i := 0; i < 2; i++ {
+		ctx, cancel := context.WithTimeout(
+			context.Background(),
+			time.Duration(constants.LlmTimeoutDurationSeconds)*time.Second,
+		)
+		defer cancel()
+
+		response, err = o.client.CreateChatCompletion(
+			ctx,
+			o.getChatCompletionRequest(llmMessages),
+		)
+		if err == nil {
+			return response, err
+		}
+	}
+
+	return response, err
+}
+
 func getClient(llmConfig commonTypes.LlmConfig) *openai.Client {
 	return openai.NewClient(llmConfig.ApiKey)
 }

backend/target/terraform/generate_code.go

@@ -1,9 +1,11 @@
 package terraform
 
 import (
+	"fmt"
 	"salami/backend/prompts/terraform/openai_gpt4"
 	backendTypes "salami/backend/types"
 	"salami/common/change_set"
+	"salami/common/logger"
 	"salami/common/symbol_table"
 	commonTypes "salami/common/types"
 
@@ -33,6 +35,8 @@ func (t *Terraform) GenerateCode(
 			semaphoreChannel <- struct{}{}
 			defer func() { <-semaphoreChannel }()
 
+			logDiffProgress(diff)
+
 			messages, err := getGenerateCodeLlmMessages(symbolTable, diff, llm)
 			if err != nil {
 				return err
@@ -73,3 +77,19 @@ func getGenerateCodeLlmMessages(
 	}
 	return messages, nil
 }
+
+func logDiffProgress(diff *commonTypes.ChangeSetDiff) {
+	var objectType string
+	var objectId string
+
+	if diff.NewObject.IsResource() {
+		objectType = "resource"
+		objectId = string(diff.NewObject.ParsedResource.LogicalName)
+	} else if diff.NewObject.IsVariable() {
+		objectType = "variable"
+		objectId = diff.NewObject.ParsedVariable.Name
+	}
+
+	message := fmt.Sprintf("🖋  Generating code for %s '%s' (diff type: %s)...", objectType, objectId, diff.DiffType)
+	logger.Verbose(message)
+}

backend/target/terraform/validate_code.go

@@ -50,13 +50,23 @@ func (t *Terraform) ValidateCode(
 	if len(changeSetRepository.Diffs) == 0 {
 		return nil
 	}
+	if retryCount == 0 {
+		logger.Verbose("🔍 Validating code...")
+	}
 	validationResults, err := generateValidationResults(newObjects)
 	if err != nil {
 		return err
 	}
-	if len(validationResults) == 0 {
+	errorCount := len(validationResults)
+	if errorCount == 0 {
+		logger.Verbose("🙌 All code is valid")
 		return nil
 	}
+	errorWord := "errors"
+	if errorCount == 1 {
+		errorWord = "error"
+	}
+	logger.Verbose(fmt.Sprintf("🔧 Found %d validation %s, fixing...", errorCount, errorWord))
 
 	if err := processValidationResults(validationResults, symbolTable, changeSetRepository, llm); err != nil {
 		return err

common/config/config.go

@@ -4,6 +4,7 @@ import (
 	"log"
 	"os"
 	"regexp"
+	"salami/common/constants"
 	"salami/common/types"
 
 	"gopkg.in/yaml.v3"
@@ -46,10 +47,17 @@ func GetTargetConfig() types.TargetConfig {
 
 func GetLlmConfig() types.LlmConfig {
 	compilerLlmConfig := getConfig().Compiler.Llm
+
+	maxConcurrentExecutions := compilerLlmConfig.MaxConcurrentExecutions
+	if maxConcurrentExecutions == 0 {
+		maxConcurrentExecutions = constants.DefaultMaxConcurrentLlmExecutions
+	}
+
 	return types.LlmConfig{
-		Provider: compilerLlmConfig.Provider,
-		Model:    compilerLlmConfig.Model,
-		ApiKey:   compilerLlmConfig.ApiKey,
+		Provider:                compilerLlmConfig.Provider,
+		Model:                   compilerLlmConfig.Model,
+		ApiKey:                  compilerLlmConfig.ApiKey,
+		MaxConcurrentExecutions: maxConcurrentExecutions,
 	}
 }
 

common/config/validate.go

@@ -62,9 +62,10 @@ type CompilerTargetConfig struct {
 }
 
 type CompilerLlmConfig struct {
-	Provider string `yaml:"provider"`
-	Model    string `yaml:"model"`
-	ApiKey   string `yaml:"api_key"`
+	Provider                string `yaml:"provider"`
+	Model                   string `yaml:"model"`
+	ApiKey                  string `yaml:"api_key"`
+	MaxConcurrentExecutions int    `yaml:"max_concurrent"`
 }
 
 func validateTarget(fl validator.FieldLevel) bool {
@@ -80,9 +81,11 @@ func validateLlm(fl validator.FieldLevel) bool {
 	if !ok {
 		return false
 	}
+
 	validLlmProvider := llmConfig.Provider == types.LlmOpenaiProvider
 	validLlmModel := llmConfig.Model == types.LlmGpt4Model
 	apiKeyExists := llmConfig.ApiKey != ""
+
 	return validLlmProvider && validLlmModel && apiKeyExists
 }
 

common/constants/constants.go

@@ -1,9 +1,13 @@
 package constants
 
-const SalamiVersion = "0.0.2"
+const SalamiVersion = "0.0.3"
 
 const SalamiFileExtension = ".sami"
 
 const TerraformFileExtension = ".tf"
 
 const MaxFixValidationErrorRetries = 2
+
+const DefaultMaxConcurrentLlmExecutions = 3
+
+const LlmTimeoutDurationSeconds = 90

common/logger/logger.go

@@ -16,6 +16,7 @@ var logger *salamiLogger
 func InitializeLogger(verbose bool) {
 	zapConfig := zap.NewDevelopmentConfig()
 	zapConfig.EncoderConfig.EncodeCaller = nil
+	zapConfig.EncoderConfig.LevelKey = ""
 
 	zapLogger, err := zapConfig.Build()
 	if err != nil {
@@ -28,7 +29,7 @@ func InitializeLogger(verbose bool) {
 	}
 }
 
-// Log logs a message always
+// Log logs the message always
 func Log(message string) {
 	if logger == nil {
 		return
@@ -38,17 +39,20 @@ func Log(message string) {
 	logger.instance.Info(message)
 }
 
-// Verbose logs a message if the verbose flag is set to true
+// Verbose logs the message if the verbose flag is set to true
 func Verbose(message string) {
 	if logger == nil {
 		return
 	}
+	if !logger.verbose {
+		return
+	}
 
 	defer logger.instance.Sync()
 	logger.instance.Info(message)
 }
 
-// Debug logs a message if the DEBUG environment variable is set to true
+// Debug logs the message if the DEBUG environment variable is set to true
 func Debug(message string) {
 	if logger == nil {
 		return

common/types/types.go

@@ -135,9 +135,10 @@ type TargetConfig struct {
 }
 
 type LlmConfig struct {
-	Provider string
-	Model    string
-	ApiKey   string
+	Provider                string
+	Model                   string
+	ApiKey                  string
+	MaxConcurrentExecutions int
 }
 
 const TerraformPlatform = "terraform"

examples/public_and_private_ecs_services/README.md

@@ -23,4 +23,4 @@ To run this example, you need:
 
 1. Note that `salami compile` will examine the `salami-lock.toml` file and the source `.sami` files, and determine which Salami objects have changed since the last compilation. To force a complete recompilation, delete the `salami-lock.toml` file. Or, you can change source `.sami` files and `salami compile` will recompile only the changed objects.
 
-2. Occassionally, OpenAI API delays responses significantly. If `salami compile` is stuck for a long time, try again later. The total compilation time varies significantly depending on the number of objects and their complexity. This project takes me about 3 minutes to compile from scratch, and a lot quicker for partial changes.
+2. Occassionally, OpenAI API delays responses significantly. If `salami compile` is stuck, try setting `compiler.llm.max_concurrent` config to a lower value.

examples/public_and_private_ecs_services/salami.yaml

@@ -5,5 +5,6 @@ compiler:
     provider: openai
     model: gpt4
     api_key: ${OPENAI_API_KEY}
+    max_concurrent: 3
   source_dir: salami
   target_dir: terraform

examples/simple_s3_bucket/README.md

@@ -23,4 +23,4 @@ To run this example, you need:
 
 1. Note that `salami compile` will examine the `salami-lock.toml` file and the source `.sami` files, and determine which Salami objects have changed since the last compilation. To force a complete recompilation, delete the `salami-lock.toml` file. Or, you can change source `.sami` files and `salami compile` will recompile only the changed objects.
 
-2. Occassionally, OpenAI API delays responses significantly. If `salami compile` is stuck for a long time, try again later. The total compilation time varies significantly depending on the number of objects and their complexity. This project takes me about 10 seconds to compile from scratch.
+2. Occassionally, OpenAI API delays responses significantly. If `salami compile` is stuck, try setting `compiler.llm.max_concurrent` config to a lower value.