Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Prd GitHub action #128

Closed
wants to merge 20 commits into from
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
20 commits
Select commit Hold shift + click to select a range
2033adb
bugfix: validate extraEnv before creating DB record
robertchoi80 Jul 31, 2023
2dad91c
bugfix. fix end date for dashboard ( pod calendar )
ktkfree Jul 31, 2023
9978ed0
Merge pull request #119 from openinfradev/TKS-791
ktkfree Aug 1, 2023
e7694b1
Merge pull request #118 from openinfradev/tks-issues-774
ktkfree Aug 1, 2023
4e3e6a7
bugfix. resource value change GB to GiB in dashboard.
ktkfree Aug 1, 2023
c031a6e
app-serving: validate name param
robertchoi80 Aug 1, 2023
9f9d0f0
app-serving: check if app name already exists
robertchoi80 Aug 1, 2023
1c20151
Merge pull request #121 from openinfradev/TKS-784
ktkfree Aug 1, 2023
ef95b14
Merge pull request #122 from openinfradev/add-name-validation
ktkfree Aug 2, 2023
3345db1
bugfix: assign transformed extraEnv to existing var
robertchoi80 Aug 2, 2023
3546f8a
bugfix. fix filtering logic in users
ktkfree Aug 1, 2023
14e357e
Merge pull request #120 from openinfradev/TKS-786
cho4036 Aug 2, 2023
0841ae5
Merge pull request #123 from openinfradev/tks-issues-796
ktkfree Aug 2, 2023
8c83023
bugfix. remove http status '204 empty'
ktkfree Aug 2, 2023
763ffab
Merge pull request #124 from openinfradev/TKS-802
ktkfree Aug 2, 2023
9efd488
bugfix. change promQL query for "pod restart" in dashboard.
ktkfree Aug 2, 2023
378c373
Merge pull request #125 from openinfradev/TKS-803
intelliguy Aug 3, 2023
b3241f7
bugfix. update kubernetes version string v1.25
ktkfree Aug 3, 2023
6e51dba
Merge pull request #126 from openinfradev/TKS-804
bluejayA Aug 3, 2023
4b316df
trivial. add github action for prd
ktkfree Aug 8, 2023
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions .github/workflows/build-and-push.yml
Original file line number Diff line number Diff line change
Expand Up @@ -51,6 +51,7 @@ jobs:
( cd cicd-manifests/${SERVICE}/overlay/ft && kustomize edit set image docker.io/sktcloud/${SERVICE}:${TAG} && git add kustomization.yaml )
elif [[ ${{github.ref}} == *"main"* ]]; then
( cd cicd-manifests/${SERVICE}/overlay/cicd && kustomize edit set image docker.io/sktcloud/${SERVICE}:${TAG} && git add kustomization.yaml )
( cd cicd-manifests/${SERVICE}/overlay/prd && kustomize edit set image docker.io/sktcloud/${SERVICE}:${TAG} && git add kustomization.yaml )
fi

cd cicd-manifests
Expand Down
18 changes: 18 additions & 0 deletions internal/delivery/http/app-serve-app.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ package http
import (
"fmt"
"net/http"
"regexp"
"strconv"
"strings"
"time"
Expand Down Expand Up @@ -129,6 +130,23 @@ func (h *AppServeAppHandler) CreateAppServeApp(w http.ResponseWriter, r *http.Re

app.AppServeAppTasks = append(app.AppServeAppTasks, task)

// Validate name param
re, _ := regexp.Compile("^[a-z][a-z0-9-]*$")
if !(re.MatchString(app.Name)) {
ErrorJSON(w, r, httpErrors.NewBadRequestError(fmt.Errorf("error: name should consist of alphanumeric characters and hyphens only"), "", ""))
return
}

exist, err := h.usecase.IsAppServeAppNameExist(organizationId, app.Name)
if err != nil {
ErrorJSON(w, r, httpErrors.NewInternalServerError(err, "", ""))
return
}
if exist {
ErrorJSON(w, r, httpErrors.NewBadRequestError(fmt.Errorf("error: name '%s' already exists.", app.Name), "", ""))
return
}

// Validate port param for springboot app
if app.AppType == "springboot" {
if task.Port == "" {
Expand Down
7 changes: 1 addition & 6 deletions internal/delivery/http/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -180,13 +180,8 @@ func (u UserHandler) List(w http.ResponseWriter, r *http.Request) {
ErrorJSON(w, r, httpErrors.NewBadRequestError(err, "", ""))
return
}
users, err := u.usecase.List(r.Context(), organizationId, pg)
users, err := u.usecase.ListWithPagination(r.Context(), organizationId, pg)
if err != nil {
if _, status := httpErrors.ErrorResponse(err); status == http.StatusNotFound {
ResponseJSON(w, r, http.StatusNoContent, domain.ListUserResponse{})
return
}

log.ErrorfWithContext(r.Context(), "error is :%s(%T)", err.Error(), err)
ErrorJSON(w, r, err)
return
Expand Down
2 changes: 1 addition & 1 deletion internal/pagination/pagination.go
Original file line number Diff line number Diff line change
Expand Up @@ -105,7 +105,7 @@ func NewPagination(urlParams *url.Values) (*Pagination, error) {
//"combinedFilter=key1,key2:value"
filterArray := strings.Split(value[0], ":")
if len(filterArray) == 2 {
keys := strings.Split(filterArray[0], ",")
keys := strings.Split(helper.ToSnakeCase(strings.Replace(filterArray[0], "[]", "", -1)), ",")
value := filterArray[1]

pg.CombinedFilter = CombinedFilter{
Expand Down
58 changes: 34 additions & 24 deletions internal/repository/user.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@ type IUserRepository interface {
Create(accountId string, organizationId string, password string, name string) (domain.User, error)
CreateWithUuid(uuid uuid.UUID, accountId string, name string, password string, email string,
department string, description string, organizationId string, roleId uuid.UUID) (domain.User, error)
List(pg *pagination.Pagination, filters ...FilterFunc) (out *[]domain.User, err error)
List(filters ...FilterFunc) (out *[]domain.User, err error)
ListWithPagination(pg *pagination.Pagination, organizationId string) (out *[]domain.User, err error)
Get(accountId string, organizationId string) (domain.User, error)
GetByUuid(userId uuid.UUID) (domain.User, error)
UpdateWithUuid(uuid uuid.UUID, accountId string, name string, password string, roleId uuid.UUID, email string,
Expand Down Expand Up @@ -143,23 +144,11 @@ func (r *UserRepository) NameFilter(name string) FilterFunc {
}
}

func (r *UserRepository) List(pg *pagination.Pagination, filters ...FilterFunc) (*[]domain.User, error) {
var res *gorm.DB
func (r *UserRepository) List(filters ...FilterFunc) (*[]domain.User, error) {
var users []User
var total int64

if pg == nil {
pg = pagination.NewDefaultPagination()
}

var res *gorm.DB
if filters == nil {
r.db.Model(&User{}).Count(&total)

pg.TotalRows = total
pg.TotalPages = int(math.Ceil(float64(total) / float64(pg.Limit)))
orderQuery := fmt.Sprintf("%s %s", pg.SortColumn, pg.SortOrder)
res = r.db.Model(&User{}).Offset(pg.GetOffset()).Limit(pg.GetLimit()).Order(orderQuery).
Preload("Organization").Preload("Role").Find(&users)
res = r.db.Model(&User{}).Preload("Organization").Preload("Role").Find(&users)
} else {
combinedFilter := func(filters ...FilterFunc) FilterFunc {
return func(user *gorm.DB) *gorm.DB {
Expand All @@ -170,14 +159,7 @@ func (r *UserRepository) List(pg *pagination.Pagination, filters ...FilterFunc)
}
}
cFunc := combinedFilter(filters...)
cFunc(r.db.Model(&User{})).Count(&total)

pg.TotalRows = total
pg.TotalPages = int(math.Ceil(float64(total) / float64(pg.Limit)))
orderQuery := fmt.Sprintf("%s %s", pg.SortColumn, pg.SortOrder)
res = cFunc(r.db.Model(&User{}).Offset(pg.GetOffset()).Limit(pg.GetLimit()).Order(orderQuery).
Preload("Organization").Preload("Role")).Find(&users)

res = cFunc(r.db.Model(&User{}).Preload("Organization").Preload("Role")).Find(&users)
}
if res.Error != nil {
log.Errorf("error is :%s(%T)", res.Error.Error(), res.Error)
Expand All @@ -194,6 +176,34 @@ func (r *UserRepository) List(pg *pagination.Pagination, filters ...FilterFunc)

return &out, nil
}

func (r *UserRepository) ListWithPagination(pg *pagination.Pagination, organizationId string) (*[]domain.User, error) {
var users []User

if pg == nil {
pg = pagination.NewDefaultPagination()
}

filterFunc := CombinedGormFilter("users", pg.GetFilters(), pg.CombinedFilter)
db := filterFunc(r.db.Model(&User{}).Where("organization_id = ?", organizationId))
db.Count(&pg.TotalRows)

pg.TotalPages = int(math.Ceil(float64(pg.TotalRows) / float64(pg.Limit)))
orderQuery := fmt.Sprintf("%s %s", pg.SortColumn, pg.SortOrder)
res := db.Preload("Organization").Preload("Role").Offset(pg.GetOffset()).Limit(pg.GetLimit()).Order(orderQuery).Find(&users)
if res.Error != nil {
log.Errorf("error is :%s(%T)", res.Error.Error(), res.Error)
return nil, res.Error
}

var out []domain.User
for _, user := range users {
out = append(out, r.reflect(user))
}

return &out, nil
}

func (r *UserRepository) Get(accountId string, organizationId string) (domain.User, error) {
user, err := r.getUserByAccountId(accountId, organizationId)
if err != nil {
Expand Down
53 changes: 26 additions & 27 deletions internal/usecase/app-serve-app.go
Original file line number Diff line number Diff line change
Expand Up @@ -89,21 +89,13 @@ func (u *AppServeAppUsecase) CreateAppServeApp(app *domain.AppServeApp) (string,
}
}

appId, taskId, err := u.repo.CreateAppServeApp(app)
if err != nil {
log.Error(err)
return "", "", errors.Wrap(err, "Failed to create app.")
}

fmt.Printf("appId = %s, taskId = %s", appId, taskId)

extEnv := app.AppServeAppTasks[0].ExtraEnv
if extEnv != "" {
/* Preprocess extraEnv param */
log.Debug("extraEnv received: ", extEnv)

tempMap := map[string]string{}
err = json.Unmarshal([]byte(extEnv), &tempMap)
err := json.Unmarshal([]byte(extEnv), &tempMap)
if err != nil {
log.Error(err)
return "", "", errors.Wrap(err, "Failed to process extraEnv param.")
Expand All @@ -118,10 +110,18 @@ func (u *AppServeAppUsecase) CreateAppServeApp(app *domain.AppServeApp) (string,
}

mJson, _ := json.Marshal(newExtEnv)
extEnv := string(mJson)
extEnv = string(mJson)
log.Debug("After transform, extraEnv: ", extEnv)
}

appId, taskId, err := u.repo.CreateAppServeApp(app)
if err != nil {
log.Error(err)
return "", "", errors.Wrap(err, "Failed to create app.")
}

fmt.Printf("appId = %s, taskId = %s", appId, taskId)

// TODO: Validate PV params

// Call argo workflow
Expand Down Expand Up @@ -379,22 +379,6 @@ func (u *AppServeAppUsecase) UpdateAppServeApp(app *domain.AppServeApp, appTask
}
}

taskId, err := u.repo.CreateTask(appTask)
if err != nil {
log.Info("taskId = ", taskId)
return "", fmt.Errorf("failed to update app-serve application. Err: %s", err)
}

// Sync new task status to the parent app
log.Info("Updating app status to 'PREPARING'..")

err = u.repo.UpdateStatus(app.ID, taskId, "PREPARING", "")
if err != nil {
log.Debug("appId = ", app.ID)
log.Debug("taskId = ", taskId)
return "", fmt.Errorf("failed to update app status on UpdateAppServeApp. Err: %s", err)
}

extEnv := appTask.ExtraEnv
if extEnv != "" {
/* Preprocess extraEnv param */
Expand All @@ -417,10 +401,25 @@ func (u *AppServeAppUsecase) UpdateAppServeApp(app *domain.AppServeApp, appTask

mJson, _ := json.Marshal(newExtEnv)
extEnv = string(mJson)

log.Debug("After transform, extraEnv: ", extEnv)
}

taskId, err := u.repo.CreateTask(appTask)
if err != nil {
log.Info("taskId = ", taskId)
return "", fmt.Errorf("failed to update app-serve application. Err: %s", err)
}

// Sync new task status to the parent app
log.Info("Updating app status to 'PREPARING'..")

err = u.repo.UpdateStatus(app.ID, taskId, "PREPARING", "")
if err != nil {
log.Debug("appId = ", app.ID)
log.Debug("taskId = ", taskId)
return "", fmt.Errorf("failed to update app status on UpdateAppServeApp. Err: %s", err)
}

// Call argo workflow
workflow := "serve-java-app"

Expand Down
8 changes: 4 additions & 4 deletions internal/usecase/auth.go
Original file line number Diff line number Diff line change
Expand Up @@ -104,7 +104,7 @@ func (u *AuthUsecase) Logout(accessToken string, organizationName string) error
return nil
}
func (u *AuthUsecase) FindId(code string, email string, userName string, organizationId string) (string, error) {
users, err := u.userRepository.List(nil, u.userRepository.OrganizationFilter(organizationId),
users, err := u.userRepository.List(u.userRepository.OrganizationFilter(organizationId),
u.userRepository.NameFilter(userName), u.userRepository.EmailFilter(email))
if err != nil && users == nil {
return "", httpErrors.NewBadRequestError(err, "A_INVALID_ID", "")
Expand Down Expand Up @@ -134,7 +134,7 @@ func (u *AuthUsecase) FindId(code string, email string, userName string, organiz
}

func (u *AuthUsecase) FindPassword(code string, accountId string, email string, userName string, organizationId string) error {
users, err := u.userRepository.List(nil, u.userRepository.OrganizationFilter(organizationId),
users, err := u.userRepository.List(u.userRepository.OrganizationFilter(organizationId),
u.userRepository.AccountIdFilter(accountId), u.userRepository.NameFilter(userName),
u.userRepository.EmailFilter(email))
if err != nil && users == nil {
Expand Down Expand Up @@ -198,10 +198,10 @@ func (u *AuthUsecase) VerifyIdentity(accountId string, email string, userName st
var err error

if accountId == "" {
users, err = u.userRepository.List(nil, u.userRepository.OrganizationFilter(organizationId),
users, err = u.userRepository.List(u.userRepository.OrganizationFilter(organizationId),
u.userRepository.NameFilter(userName), u.userRepository.EmailFilter(email))
} else {
users, err = u.userRepository.List(nil, u.userRepository.OrganizationFilter(organizationId),
users, err = u.userRepository.List(u.userRepository.OrganizationFilter(organizationId),
u.userRepository.AccountIdFilter(accountId), u.userRepository.NameFilter(userName),
u.userRepository.EmailFilter(email))
}
Expand Down
22 changes: 11 additions & 11 deletions internal/usecase/dashboard.go
Original file line number Diff line number Diff line change
Expand Up @@ -185,36 +185,36 @@ func (u *DashboardUsecase) GetResources(ctx context.Context, organizationId stri
if err != nil {
return out, err
}
memory := 0
memory := float64(0)
for _, val := range result.Data.Result {
memoryVal, err := strconv.Atoi(val.Value[1].(string))
if err != nil {
continue
}
if memoryVal > 0 {
memoryVal = memoryVal / 1024 / 1024 / 1024
memory = memory + memoryVal
memory_ := float64(memoryVal) / float64(1024) / float64(1024) / float64(1024)
memory = memory + memory_
}
}
out.Memory = fmt.Sprintf("%d GB", memory)
out.Memory = fmt.Sprintf("%v GiB", math.Round(memory))

// Storage
result, err = thanosClient.Get("sum by (taco_cluster) (kubelet_volume_stats_capacity_bytes)")
if err != nil {
return out, err
}
storage := 0
storage := float64(0)
for _, val := range result.Data.Result {
storageVal, err := strconv.Atoi(val.Value[1].(string))
if err != nil {
continue
}
if storageVal > 0 {
storageVal = storageVal / 1024 / 1024 / 1024
storage = storage + storageVal
storage_ := float64(storageVal) / float64(1024) / float64(1024) / float64(1024)
storage = storage + storage_
}
}
out.Storage = fmt.Sprintf("%d GB", storage)
out.Storage = fmt.Sprintf("%v GiB", math.Round(storage))

return
}
Expand Down Expand Up @@ -267,7 +267,7 @@ func (u *DashboardUsecase) getChartFromPrometheus(organizationId string, chartTy
query = "avg by (taco_cluster) (sum(node_memory_MemTotal_bytes - node_memory_MemAvailable_bytes) by (taco_cluster) / sum(node_memory_MemTotal_bytes) by (taco_cluster))"

case domain.ChartType_POD.String():
query = "avg by (taco_cluster) (increase(kube_pod_container_status_restarts_total{namespace!=\"kube-system\"}[1h]))"
query = "sum by (taco_cluster) (changes(kube_pod_container_status_restarts_total{namespace!=\"kube-system\"}[1h]))"

case domain.ChartType_TRAFFIC.String():
query = "avg by (taco_cluster) (rate(container_network_receive_bytes_total[1h]))"
Expand All @@ -277,7 +277,7 @@ func (u *DashboardUsecase) getChartFromPrometheus(organizationId string, chartTy
yearInt, _ := strconv.Atoi(year)
monthInt, _ := strconv.Atoi(month)
startDate := time.Date(yearInt, time.Month(monthInt), 1, 0, 0, 0, 0, time.UTC)
endDate := startDate.Add(time.Hour * 24 * 30)
endDate := time.Date(yearInt, time.Month(monthInt+1), 1, 0, 0, 0, 0, time.UTC)

if now.Year() < yearInt {
return res, fmt.Errorf("Invalid year")
Expand Down Expand Up @@ -340,7 +340,7 @@ func (u *DashboardUsecase) getChartFromPrometheus(organizationId string, chartTy
return domain.DashboardChart{}, fmt.Errorf("No data")
}

result, err := thanosClient.FetchRange(query, int(now.Unix())-durationSec, int(now.Unix()), intervalSec)
result, err := thanosClient.FetchRange(query, int(now.Unix())-durationSec, int(now.Unix())+10000, intervalSec)
if err != nil {
return res, err
}
Expand Down
Loading
Loading