diff --git a/Makefile b/Makefile index 3dcab4a9..0eb6e7e3 100644 --- a/Makefile +++ b/Makefile @@ -33,6 +33,7 @@ default: build build-static: ## Build garm statically @echo Building garm docker build --tag $(IMAGE_TAG) -f Dockerfile.build-static . + mkdir -p build docker run --rm -e USER_ID=$(USER_ID) -e GARM_REF=$(GARM_REF) -e USER_GROUP=$(USER_GROUP) -v $(PWD)/build:/build/output:z $(IMAGE_TAG) /build-static.sh @echo Binaries are available in $(PWD)/build diff --git a/README.md b/README.md index 2ee51ed0..9536b8fb 100644 --- a/README.md +++ b/README.md @@ -10,7 +10,7 @@ The goal of ```GARM``` is to be simple to set up, simple to configure and simple GARM supports creating pools on either GitHub itself or on your own deployment of [GitHub Enterprise Server](https://docs.github.com/en/enterprise-server@3.5/admin/overview/about-github-enterprise-server). For instructions on how to use ```GARM``` with GHE, see the [credentials](/doc/github_credentials.md) section of the documentation. -Through the use of providers, `GARM` can create runners in a variety of environments using the same `GARM` instance. Want to create pools of runners in your OpenStack cloud, your Azure cloud and your Kubernetes cluster? No problem! Just install the appropriate providers, configure them in `GARM` and you're good to go. Create zero-runner pools for instances with high costs (large VMs, GPU enabled instances, etc) and have them spin up on demand, or create large pools of k8s backed runners that can be used for your CI/CD pipelines at a moment's notice. You can mix them up and create pools in any combination of providers or resource allocations you want. +Through the use of providers, `GARM` can create runners in a variety of environments using the same `GARM` instance. Whether you want to create pools of runners in your OpenStack cloud, your Azure cloud and your Kubernetes cluster, that is easily achieved by just installing the appropriate providers, configuring them in `GARM` and creating pools that use them. You can create zero-runner pools for instances with high costs (large VMs, GPU enabled instances, etc) and have them spin up on demand, or you can create large pools of k8s backed runners that can be used for your CI/CD pipelines at a moment's notice. You can mix them up and create pools in any combination of providers or resource allocations you want. :warning: **Important note**: The README and documentation in the `main` branch are relevant to the not yet released code that is present in `main`. Following the documentation from the `main` branch for a stable release of GARM, may lead to errors. To view the documentation for the latest stable release, please switch to the appropriate tag. For information about setting up `v0.1.4`, please refer to the [v0.1.4 tag](https://github.com/cloudbase/garm/tree/v0.1.4) diff --git a/apiserver/controllers/controllers.go b/apiserver/controllers/controllers.go index 56745b82..556892f3 100644 --- a/apiserver/controllers/controllers.go +++ b/apiserver/controllers/controllers.go @@ -391,3 +391,42 @@ func (a *APIController) ControllerInfoHandler(w http.ResponseWriter, r *http.Req slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response") } } + +// swagger:route PUT /controller controller UpdateController +// +// Update controller. +// +// Parameters: +// + name: Body +// description: Parameters used when updating the controller. +// type: UpdateControllerParams +// in: body +// required: true +// +// Responses: +// 200: ControllerInfo +// 400: APIErrorResponse +func (a *APIController) UpdateControllerHandler(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + var updateParams runnerParams.UpdateControllerParams + if err := json.NewDecoder(r.Body).Decode(&updateParams); err != nil { + handleError(ctx, w, gErrors.ErrBadRequest) + return + } + + if err := updateParams.Validate(); err != nil { + handleError(ctx, w, err) + return + } + + info, err := a.r.UpdateController(ctx, updateParams) + if err != nil { + handleError(ctx, w, err) + return + } + + w.Header().Set("Content-Type", "application/json") + if err := json.NewEncoder(w).Encode(info); err != nil { + slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response") + } +} diff --git a/apiserver/params/params.go b/apiserver/params/params.go index 23283a07..6e46190e 100644 --- a/apiserver/params/params.go +++ b/apiserver/params/params.go @@ -36,4 +36,9 @@ var ( Error: "init_required", Details: "Missing superuser", } + // URLsRequired is returned if the controller does not have the required URLs + URLsRequired = APIErrorResponse{ + Error: "urls_required", + Details: "Missing required URLs. Make sure you update the metadata, callback and webhook URLs", + } ) diff --git a/apiserver/routers/routers.go b/apiserver/routers/routers.go index 5683a8c2..95d31518 100644 --- a/apiserver/routers/routers.go +++ b/apiserver/routers/routers.go @@ -100,7 +100,7 @@ func requestLogger(h http.Handler) http.Handler { }) } -func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware, instanceMiddleware auth.Middleware, manageWebhooks bool) *mux.Router { +func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware, urlsRequiredMiddleware, instanceMiddleware auth.Middleware, manageWebhooks bool) *mux.Router { router := mux.NewRouter() router.Use(requestLogger) @@ -152,11 +152,38 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware authRouter.Handle("/{login:login\\/?}", http.HandlerFunc(han.LoginHandler)).Methods("POST", "OPTIONS") authRouter.Use(initMiddleware.Middleware) + ////////////////////////// + // Controller endpoints // + ////////////////////////// + controllerRouter := apiSubRouter.PathPrefix("/controller").Subrouter() + // The controller endpoints allow us to get information about the controller and update the URL endpoints. + // This endpoint must not be guarded by the urlsRequiredMiddleware as that would prevent the user from + // updating the URLs. + controllerRouter.Use(initMiddleware.Middleware) + controllerRouter.Use(authMiddleware.Middleware) + controllerRouter.Use(auth.AdminRequiredMiddleware) + // Get controller info + controllerRouter.Handle("/", http.HandlerFunc(han.ControllerInfoHandler)).Methods("GET", "OPTIONS") + controllerRouter.Handle("", http.HandlerFunc(han.ControllerInfoHandler)).Methods("GET", "OPTIONS") + // Update controller + controllerRouter.Handle("/", http.HandlerFunc(han.UpdateControllerHandler)).Methods("PUT", "OPTIONS") + controllerRouter.Handle("", http.HandlerFunc(han.UpdateControllerHandler)).Methods("PUT", "OPTIONS") + + //////////////////////////////////// + // API router for everything else // + //////////////////////////////////// apiRouter := apiSubRouter.PathPrefix("").Subrouter() apiRouter.Use(initMiddleware.Middleware) + // all endpoints except the controller endpoint should return an error + // if the required metadata, callback and webhook URLs are not set. + apiRouter.Use(urlsRequiredMiddleware.Middleware) apiRouter.Use(authMiddleware.Middleware) apiRouter.Use(auth.AdminRequiredMiddleware) + // Legacy controller path + apiRouter.Handle("/controller-info/", http.HandlerFunc(han.ControllerInfoHandler)).Methods("GET", "OPTIONS") + apiRouter.Handle("/controller-info", http.HandlerFunc(han.ControllerInfoHandler)).Methods("GET", "OPTIONS") + // Metrics Token apiRouter.Handle("/metrics-token/", http.HandlerFunc(han.MetricsTokenHandler)).Methods("GET", "OPTIONS") apiRouter.Handle("/metrics-token", http.HandlerFunc(han.MetricsTokenHandler)).Methods("GET", "OPTIONS") @@ -343,10 +370,6 @@ func NewAPIRouter(han *controllers.APIController, authMiddleware, initMiddleware apiRouter.Handle("/providers/", http.HandlerFunc(han.ListProviders)).Methods("GET", "OPTIONS") apiRouter.Handle("/providers", http.HandlerFunc(han.ListProviders)).Methods("GET", "OPTIONS") - // Controller info - apiRouter.Handle("/controller-info/", http.HandlerFunc(han.ControllerInfoHandler)).Methods("GET", "OPTIONS") - apiRouter.Handle("/controller-info", http.HandlerFunc(han.ControllerInfoHandler)).Methods("GET", "OPTIONS") - ////////////////////// // Github Endpoints // ////////////////////// diff --git a/apiserver/swagger-models.yaml b/apiserver/swagger-models.yaml index b9ab5670..88c6bd8d 100644 --- a/apiserver/swagger-models.yaml +++ b/apiserver/swagger-models.yaml @@ -278,3 +278,10 @@ definitions: import: package: github.com/cloudbase/garm/params alias: garm_params + UpdateControllerParams: + type: object + x-go-type: + type: UpdateControllerParams + import: + package: github.com/cloudbase/garm/params + alias: garm_params diff --git a/apiserver/swagger.yaml b/apiserver/swagger.yaml index afed3747..42c573f0 100644 --- a/apiserver/swagger.yaml +++ b/apiserver/swagger.yaml @@ -244,6 +244,13 @@ definitions: alias: garm_params package: github.com/cloudbase/garm/params type: Repository + UpdateControllerParams: + type: object + x-go-type: + import: + alias: garm_params + package: github.com/cloudbase/garm/params + type: UpdateControllerParams UpdateEntityParams: type: object x-go-type: @@ -311,6 +318,30 @@ paths: summary: Logs in a user and returns a JWT token. tags: - login + /controller: + put: + operationId: UpdateController + parameters: + - description: Parameters used when updating the controller. + in: body + name: Body + required: true + schema: + $ref: '#/definitions/UpdateControllerParams' + description: Parameters used when updating the controller. + type: object + responses: + "200": + description: ControllerInfo + schema: + $ref: '#/definitions/ControllerInfo' + "400": + description: APIErrorResponse + schema: + $ref: '#/definitions/APIErrorResponse' + summary: Update controller. + tags: + - controller /controller-info: get: operationId: ControllerInfo diff --git a/auth/init_required.go b/auth/init_required.go index 7dcc655b..6b369a6c 100644 --- a/auth/init_required.go +++ b/auth/init_required.go @@ -51,3 +51,30 @@ func (i *initRequired) Middleware(next http.Handler) http.Handler { next.ServeHTTP(w, r.WithContext(ctx)) }) } + +func NewUrlsRequiredMiddleware(store common.Store) (Middleware, error) { + return &urlsRequired{ + store: store, + }, nil +} + +type urlsRequired struct { + store common.Store +} + +func (u *urlsRequired) Middleware(next http.Handler) http.Handler { + return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + ctx := r.Context() + ctrlInfo, err := u.store.ControllerInfo() + if err != nil || ctrlInfo.WebhookURL == "" || ctrlInfo.MetadataURL == "" || ctrlInfo.CallbackURL == "" { + w.Header().Add("Content-Type", "application/json") + w.WriteHeader(http.StatusConflict) + if err := json.NewEncoder(w).Encode(params.URLsRequired); err != nil { + slog.With(slog.Any("error", err)).ErrorContext(ctx, "failed to encode response") + } + return + } + + next.ServeHTTP(w, r.WithContext(ctx)) + }) +} diff --git a/client/controller/controller_client.go b/client/controller/controller_client.go new file mode 100644 index 00000000..cf6cde1a --- /dev/null +++ b/client/controller/controller_client.go @@ -0,0 +1,106 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package controller + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "fmt" + + "github.com/go-openapi/runtime" + httptransport "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" +) + +// New creates a new controller API client. +func New(transport runtime.ClientTransport, formats strfmt.Registry) ClientService { + return &Client{transport: transport, formats: formats} +} + +// New creates a new controller API client with basic auth credentials. +// It takes the following parameters: +// - host: http host (github.com). +// - basePath: any base path for the API client ("/v1", "/v3"). +// - scheme: http scheme ("http", "https"). +// - user: user for basic authentication header. +// - password: password for basic authentication header. +func NewClientWithBasicAuth(host, basePath, scheme, user, password string) ClientService { + transport := httptransport.New(host, basePath, []string{scheme}) + transport.DefaultAuthentication = httptransport.BasicAuth(user, password) + return &Client{transport: transport, formats: strfmt.Default} +} + +// New creates a new controller API client with a bearer token for authentication. +// It takes the following parameters: +// - host: http host (github.com). +// - basePath: any base path for the API client ("/v1", "/v3"). +// - scheme: http scheme ("http", "https"). +// - bearerToken: bearer token for Bearer authentication header. +func NewClientWithBearerToken(host, basePath, scheme, bearerToken string) ClientService { + transport := httptransport.New(host, basePath, []string{scheme}) + transport.DefaultAuthentication = httptransport.BearerToken(bearerToken) + return &Client{transport: transport, formats: strfmt.Default} +} + +/* +Client for controller API +*/ +type Client struct { + transport runtime.ClientTransport + formats strfmt.Registry +} + +// ClientOption may be used to customize the behavior of Client methods. +type ClientOption func(*runtime.ClientOperation) + +// ClientService is the interface for Client methods +type ClientService interface { + UpdateController(params *UpdateControllerParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*UpdateControllerOK, error) + + SetTransport(transport runtime.ClientTransport) +} + +/* +UpdateController updates controller +*/ +func (a *Client) UpdateController(params *UpdateControllerParams, authInfo runtime.ClientAuthInfoWriter, opts ...ClientOption) (*UpdateControllerOK, error) { + // TODO: Validate the params before sending + if params == nil { + params = NewUpdateControllerParams() + } + op := &runtime.ClientOperation{ + ID: "UpdateController", + Method: "PUT", + PathPattern: "/controller", + ProducesMediaTypes: []string{"application/json"}, + ConsumesMediaTypes: []string{"application/json"}, + Schemes: []string{"http"}, + Params: params, + Reader: &UpdateControllerReader{formats: a.formats}, + AuthInfo: authInfo, + Context: params.Context, + Client: params.HTTPClient, + } + for _, opt := range opts { + opt(op) + } + + result, err := a.transport.Submit(op) + if err != nil { + return nil, err + } + success, ok := result.(*UpdateControllerOK) + if ok { + return success, nil + } + // unexpected success response + // safeguard: normally, absent a default response, unknown success responses return an error above: so this is a codegen issue + msg := fmt.Sprintf("unexpected success response for UpdateController: API contract not enforced by server. Client expected to get an error, but got: %T", result) + panic(msg) +} + +// SetTransport changes the transport on the client +func (a *Client) SetTransport(transport runtime.ClientTransport) { + a.transport = transport +} diff --git a/client/controller/update_controller_parameters.go b/client/controller/update_controller_parameters.go new file mode 100644 index 00000000..a0705d60 --- /dev/null +++ b/client/controller/update_controller_parameters.go @@ -0,0 +1,151 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package controller + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "context" + "net/http" + "time" + + "github.com/go-openapi/errors" + "github.com/go-openapi/runtime" + cr "github.com/go-openapi/runtime/client" + "github.com/go-openapi/strfmt" + + garm_params "github.com/cloudbase/garm/params" +) + +// NewUpdateControllerParams creates a new UpdateControllerParams object, +// with the default timeout for this client. +// +// Default values are not hydrated, since defaults are normally applied by the API server side. +// +// To enforce default values in parameter, use SetDefaults or WithDefaults. +func NewUpdateControllerParams() *UpdateControllerParams { + return &UpdateControllerParams{ + timeout: cr.DefaultTimeout, + } +} + +// NewUpdateControllerParamsWithTimeout creates a new UpdateControllerParams object +// with the ability to set a timeout on a request. +func NewUpdateControllerParamsWithTimeout(timeout time.Duration) *UpdateControllerParams { + return &UpdateControllerParams{ + timeout: timeout, + } +} + +// NewUpdateControllerParamsWithContext creates a new UpdateControllerParams object +// with the ability to set a context for a request. +func NewUpdateControllerParamsWithContext(ctx context.Context) *UpdateControllerParams { + return &UpdateControllerParams{ + Context: ctx, + } +} + +// NewUpdateControllerParamsWithHTTPClient creates a new UpdateControllerParams object +// with the ability to set a custom HTTPClient for a request. +func NewUpdateControllerParamsWithHTTPClient(client *http.Client) *UpdateControllerParams { + return &UpdateControllerParams{ + HTTPClient: client, + } +} + +/* +UpdateControllerParams contains all the parameters to send to the API endpoint + + for the update controller operation. + + Typically these are written to a http.Request. +*/ +type UpdateControllerParams struct { + + /* Body. + + Parameters used when updating the controller. + */ + Body garm_params.UpdateControllerParams + + timeout time.Duration + Context context.Context + HTTPClient *http.Client +} + +// WithDefaults hydrates default values in the update controller params (not the query body). +// +// All values with no default are reset to their zero value. +func (o *UpdateControllerParams) WithDefaults() *UpdateControllerParams { + o.SetDefaults() + return o +} + +// SetDefaults hydrates default values in the update controller params (not the query body). +// +// All values with no default are reset to their zero value. +func (o *UpdateControllerParams) SetDefaults() { + // no default values defined for this parameter +} + +// WithTimeout adds the timeout to the update controller params +func (o *UpdateControllerParams) WithTimeout(timeout time.Duration) *UpdateControllerParams { + o.SetTimeout(timeout) + return o +} + +// SetTimeout adds the timeout to the update controller params +func (o *UpdateControllerParams) SetTimeout(timeout time.Duration) { + o.timeout = timeout +} + +// WithContext adds the context to the update controller params +func (o *UpdateControllerParams) WithContext(ctx context.Context) *UpdateControllerParams { + o.SetContext(ctx) + return o +} + +// SetContext adds the context to the update controller params +func (o *UpdateControllerParams) SetContext(ctx context.Context) { + o.Context = ctx +} + +// WithHTTPClient adds the HTTPClient to the update controller params +func (o *UpdateControllerParams) WithHTTPClient(client *http.Client) *UpdateControllerParams { + o.SetHTTPClient(client) + return o +} + +// SetHTTPClient adds the HTTPClient to the update controller params +func (o *UpdateControllerParams) SetHTTPClient(client *http.Client) { + o.HTTPClient = client +} + +// WithBody adds the body to the update controller params +func (o *UpdateControllerParams) WithBody(body garm_params.UpdateControllerParams) *UpdateControllerParams { + o.SetBody(body) + return o +} + +// SetBody adds the body to the update controller params +func (o *UpdateControllerParams) SetBody(body garm_params.UpdateControllerParams) { + o.Body = body +} + +// WriteToRequest writes these params to a swagger request +func (o *UpdateControllerParams) WriteToRequest(r runtime.ClientRequest, reg strfmt.Registry) error { + + if err := r.SetTimeout(o.timeout); err != nil { + return err + } + var res []error + if err := r.SetBodyParam(o.Body); err != nil { + return err + } + + if len(res) > 0 { + return errors.CompositeValidationError(res...) + } + return nil +} diff --git a/client/controller/update_controller_responses.go b/client/controller/update_controller_responses.go new file mode 100644 index 00000000..f555a78e --- /dev/null +++ b/client/controller/update_controller_responses.go @@ -0,0 +1,179 @@ +// Code generated by go-swagger; DO NOT EDIT. + +package controller + +// This file was generated by the swagger tool. +// Editing this file might prove futile when you re-run the swagger generate command + +import ( + "encoding/json" + "fmt" + "io" + + "github.com/go-openapi/runtime" + "github.com/go-openapi/strfmt" + + apiserver_params "github.com/cloudbase/garm/apiserver/params" + garm_params "github.com/cloudbase/garm/params" +) + +// UpdateControllerReader is a Reader for the UpdateController structure. +type UpdateControllerReader struct { + formats strfmt.Registry +} + +// ReadResponse reads a server response into the received o. +func (o *UpdateControllerReader) ReadResponse(response runtime.ClientResponse, consumer runtime.Consumer) (interface{}, error) { + switch response.Code() { + case 200: + result := NewUpdateControllerOK() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return result, nil + case 400: + result := NewUpdateControllerBadRequest() + if err := result.readResponse(response, consumer, o.formats); err != nil { + return nil, err + } + return nil, result + default: + return nil, runtime.NewAPIError("[PUT /controller] UpdateController", response, response.Code()) + } +} + +// NewUpdateControllerOK creates a UpdateControllerOK with default headers values +func NewUpdateControllerOK() *UpdateControllerOK { + return &UpdateControllerOK{} +} + +/* +UpdateControllerOK describes a response with status code 200, with default header values. + +ControllerInfo +*/ +type UpdateControllerOK struct { + Payload garm_params.ControllerInfo +} + +// IsSuccess returns true when this update controller o k response has a 2xx status code +func (o *UpdateControllerOK) IsSuccess() bool { + return true +} + +// IsRedirect returns true when this update controller o k response has a 3xx status code +func (o *UpdateControllerOK) IsRedirect() bool { + return false +} + +// IsClientError returns true when this update controller o k response has a 4xx status code +func (o *UpdateControllerOK) IsClientError() bool { + return false +} + +// IsServerError returns true when this update controller o k response has a 5xx status code +func (o *UpdateControllerOK) IsServerError() bool { + return false +} + +// IsCode returns true when this update controller o k response a status code equal to that given +func (o *UpdateControllerOK) IsCode(code int) bool { + return code == 200 +} + +// Code gets the status code for the update controller o k response +func (o *UpdateControllerOK) Code() int { + return 200 +} + +func (o *UpdateControllerOK) Error() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[PUT /controller][%d] updateControllerOK %s", 200, payload) +} + +func (o *UpdateControllerOK) String() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[PUT /controller][%d] updateControllerOK %s", 200, payload) +} + +func (o *UpdateControllerOK) GetPayload() garm_params.ControllerInfo { + return o.Payload +} + +func (o *UpdateControllerOK) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} + +// NewUpdateControllerBadRequest creates a UpdateControllerBadRequest with default headers values +func NewUpdateControllerBadRequest() *UpdateControllerBadRequest { + return &UpdateControllerBadRequest{} +} + +/* +UpdateControllerBadRequest describes a response with status code 400, with default header values. + +APIErrorResponse +*/ +type UpdateControllerBadRequest struct { + Payload apiserver_params.APIErrorResponse +} + +// IsSuccess returns true when this update controller bad request response has a 2xx status code +func (o *UpdateControllerBadRequest) IsSuccess() bool { + return false +} + +// IsRedirect returns true when this update controller bad request response has a 3xx status code +func (o *UpdateControllerBadRequest) IsRedirect() bool { + return false +} + +// IsClientError returns true when this update controller bad request response has a 4xx status code +func (o *UpdateControllerBadRequest) IsClientError() bool { + return true +} + +// IsServerError returns true when this update controller bad request response has a 5xx status code +func (o *UpdateControllerBadRequest) IsServerError() bool { + return false +} + +// IsCode returns true when this update controller bad request response a status code equal to that given +func (o *UpdateControllerBadRequest) IsCode(code int) bool { + return code == 400 +} + +// Code gets the status code for the update controller bad request response +func (o *UpdateControllerBadRequest) Code() int { + return 400 +} + +func (o *UpdateControllerBadRequest) Error() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[PUT /controller][%d] updateControllerBadRequest %s", 400, payload) +} + +func (o *UpdateControllerBadRequest) String() string { + payload, _ := json.Marshal(o.Payload) + return fmt.Sprintf("[PUT /controller][%d] updateControllerBadRequest %s", 400, payload) +} + +func (o *UpdateControllerBadRequest) GetPayload() apiserver_params.APIErrorResponse { + return o.Payload +} + +func (o *UpdateControllerBadRequest) readResponse(response runtime.ClientResponse, consumer runtime.Consumer, formats strfmt.Registry) error { + + // response payload + if err := consumer.Consume(response.Body(), &o.Payload); err != nil && err != io.EOF { + return err + } + + return nil +} diff --git a/client/garm_api_client.go b/client/garm_api_client.go index 597eab26..cbc65dfc 100644 --- a/client/garm_api_client.go +++ b/client/garm_api_client.go @@ -10,6 +10,7 @@ import ( httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + "github.com/cloudbase/garm/client/controller" "github.com/cloudbase/garm/client/controller_info" "github.com/cloudbase/garm/client/credentials" "github.com/cloudbase/garm/client/endpoints" @@ -67,6 +68,7 @@ func New(transport runtime.ClientTransport, formats strfmt.Registry) *GarmAPI { cli := new(GarmAPI) cli.Transport = transport + cli.Controller = controller.New(transport, formats) cli.ControllerInfo = controller_info.New(transport, formats) cli.Credentials = credentials.New(transport, formats) cli.Endpoints = endpoints.New(transport, formats) @@ -124,6 +126,8 @@ func (cfg *TransportConfig) WithSchemes(schemes []string) *TransportConfig { // GarmAPI is a client for garm API type GarmAPI struct { + Controller controller.ClientService + ControllerInfo controller_info.ClientService Credentials credentials.ClientService @@ -156,6 +160,7 @@ type GarmAPI struct { // SetTransport changes the transport on the client and all its subresources func (c *GarmAPI) SetTransport(transport runtime.ClientTransport) { c.Transport = transport + c.Controller.SetTransport(transport) c.ControllerInfo.SetTransport(transport) c.Credentials.SetTransport(transport) c.Endpoints.SetTransport(transport) diff --git a/cmd/garm-cli/cmd/controller.go b/cmd/garm-cli/cmd/controller.go new file mode 100644 index 00000000..2141121b --- /dev/null +++ b/cmd/garm-cli/cmd/controller.go @@ -0,0 +1,173 @@ +// Copyright 2023 Cloudbase Solutions SRL +// +// Licensed under the Apache License, Version 2.0 (the "License"); you may +// not use this file except in compliance with the License. You may obtain +// a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +// License for the specific language governing permissions and limitations +// under the License. + +package cmd + +import ( + "fmt" + + "github.com/jedib0t/go-pretty/v6/table" + "github.com/spf13/cobra" + + apiClientController "github.com/cloudbase/garm/client/controller" + apiClientControllerInfo "github.com/cloudbase/garm/client/controller_info" + "github.com/cloudbase/garm/params" +) + +var controllerCmd = &cobra.Command{ + Use: "controller", + Aliases: []string{"controller-info"}, + SilenceUsage: true, + Short: "Controller operations", + Long: `Query or update information about the current controller.`, + Run: nil, +} + +var controllerShowCmd = &cobra.Command{ + Use: "show", + Short: "Show information", + Long: `Show information about the current controller.`, + SilenceUsage: true, + RunE: func(_ *cobra.Command, _ []string) error { + if needsInit { + return errNeedsInitError + } + + showInfo := apiClientControllerInfo.NewControllerInfoParams() + response, err := apiCli.ControllerInfo.ControllerInfo(showInfo, authToken) + if err != nil { + return err + } + return formatInfo(response.Payload) + }, +} + +var controllerUpdateCmd = &cobra.Command{ + Use: "update", + Short: "Update controller information", + Long: `Update information about the current controller. + +Warning: Dragons ahead, please read carefully. + +Changing the URLs for the controller metadata, callback and webhooks, will +impact the controller's ability to manage webhooks and runners. + +As GARM can be set up behind a reverse proxy or through several layers of +network address translation or load balancing, we need to explicitly tell +GARM how to reach each of these URLs. Internally, GARM sets up API endpoints +as follows: + + * /webhooks - the base URL for the webhooks. Github needs to reach this URL. + * /api/v1/metadata - the metadata URL. Your runners need to be able to reach this URL. + * /api/v1/callbacks - the callback URL. Your runners need to be able to reach this URL. + +You need to expose these endpoints to the interested parties (github or +your runners), then you need to update the controller with the URLs you set up. + +For example, if you set the webhooks URL in your reverse proxy to +https://garm.example.com/garm-hooks, this still needs to point to the "/webhooks" +URL in the GARM backend, but in the controller info you need to set the URL to +https://garm.example.com/garm-hooks using: + + garm-cli controller update --webhook-url=https://garm.example.com/garm-hooks + +If you expose GARM to the outside world directly, or if you don't rewrite the URLs +above in your reverse proxy config, use the above 3 endpoints without change, +substituting garm.example.com with the correct hostname or IP address. + +In most cases, you will have a GARM backend (say 192.168.100.10) and a reverse +proxy in front of it exposed as https://garm.example.com. If you don't rewrite +the URLs in the reverse proxy, and you just point to your backend, you can set +up the GARM controller URLs as: + + garm-cli controller update \ + --webhook-url=https://garm.example.com/webhooks \ + --metadata-url=https://garm.example.com/api/v1/metadata \ + --callback-url=https://garm.example.com/api/v1/callbacks +`, + SilenceUsage: true, + RunE: func(cmd *cobra.Command, _ []string) error { + if needsInit { + return errNeedsInitError + } + + params := params.UpdateControllerParams{} + if cmd.Flags().Changed("metadata-url") { + params.MetadataURL = &metadataURL + } + if cmd.Flags().Changed("callback-url") { + params.CallbackURL = &callbackURL + } + if cmd.Flags().Changed("webhook-url") { + params.WebhookURL = &webhookURL + } + + if params.WebhookURL == nil && params.MetadataURL == nil && params.CallbackURL == nil { + cmd.Help() + return fmt.Errorf("at least one of metadata-url, callback-url or webhook-url must be provided") + } + + updateUrlsReq := apiClientController.NewUpdateControllerParams() + updateUrlsReq.Body = params + + info, err := apiCli.Controller.UpdateController(updateUrlsReq, authToken) + if err != nil { + return fmt.Errorf("error updating controller: %w", err) + } + formatInfo(info.Payload) + return nil + }, +} + +func renderControllerInfoTable(info params.ControllerInfo) string { + t := table.NewWriter() + header := table.Row{"Field", "Value"} + + if info.WebhookURL == "" { + info.WebhookURL = "N/A" + } + + if info.ControllerWebhookURL == "" { + info.ControllerWebhookURL = "N/A" + } + + t.AppendHeader(header) + t.AppendRow(table.Row{"Controller ID", info.ControllerID}) + if info.Hostname != "" { + t.AppendRow(table.Row{"Hostname", info.Hostname}) + } + t.AppendRow(table.Row{"Metadata URL", info.MetadataURL}) + t.AppendRow(table.Row{"Callback URL", info.CallbackURL}) + t.AppendRow(table.Row{"Webhook Base URL", info.WebhookURL}) + t.AppendRow(table.Row{"Controller Webhook URL", info.ControllerWebhookURL}) + return t.Render() +} + +func formatInfo(info params.ControllerInfo) error { + fmt.Println(renderControllerInfoTable(info)) + return nil +} + +func init() { + controllerUpdateCmd.Flags().StringVarP(&metadataURL, "metadata-url", "m", "", "The metadata URL for the controller (ie. https://garm.example.com/api/v1/metadata)") + controllerUpdateCmd.Flags().StringVarP(&callbackURL, "callback-url", "c", "", "The callback URL for the controller (ie. https://garm.example.com/api/v1/callbacks)") + controllerUpdateCmd.Flags().StringVarP(&webhookURL, "webhook-url", "w", "", "The webhook URL for the controller (ie. https://garm.example.com/webhooks)") + + controllerCmd.AddCommand( + controllerShowCmd, + controllerUpdateCmd, + ) + + rootCmd.AddCommand(controllerCmd) +} diff --git a/cmd/garm-cli/cmd/controller_info.go b/cmd/garm-cli/cmd/controller_info.go deleted file mode 100644 index 67ef2b86..00000000 --- a/cmd/garm-cli/cmd/controller_info.go +++ /dev/null @@ -1,84 +0,0 @@ -// Copyright 2023 Cloudbase Solutions SRL -// -// Licensed under the Apache License, Version 2.0 (the "License"); you may -// not use this file except in compliance with the License. You may obtain -// a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, WITHOUT -// WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the -// License for the specific language governing permissions and limitations -// under the License. - -package cmd - -import ( - "fmt" - - "github.com/jedib0t/go-pretty/v6/table" - "github.com/spf13/cobra" - - apiClientControllerInfo "github.com/cloudbase/garm/client/controller_info" - "github.com/cloudbase/garm/params" -) - -var infoCmd = &cobra.Command{ - Use: "controller-info", - SilenceUsage: true, - Short: "Information about controller", - Long: `Query information about the current controller.`, - Run: nil, -} - -var infoShowCmd = &cobra.Command{ - Use: "show", - Short: "Show information", - Long: `Show information about the current controller.`, - SilenceUsage: true, - RunE: func(_ *cobra.Command, _ []string) error { - if needsInit { - return errNeedsInitError - } - - showInfo := apiClientControllerInfo.NewControllerInfoParams() - response, err := apiCli.ControllerInfo.ControllerInfo(showInfo, authToken) - if err != nil { - return err - } - return formatInfo(response.Payload) - }, -} - -func formatInfo(info params.ControllerInfo) error { - t := table.NewWriter() - header := table.Row{"Field", "Value"} - - if info.WebhookURL == "" { - info.WebhookURL = "N/A" - } - - if info.ControllerWebhookURL == "" { - info.ControllerWebhookURL = "N/A" - } - - t.AppendHeader(header) - t.AppendRow(table.Row{"Controller ID", info.ControllerID}) - t.AppendRow(table.Row{"Hostname", info.Hostname}) - t.AppendRow(table.Row{"Metadata URL", info.MetadataURL}) - t.AppendRow(table.Row{"Callback URL", info.CallbackURL}) - t.AppendRow(table.Row{"Webhook Base URL", info.WebhookURL}) - t.AppendRow(table.Row{"Controller Webhook URL", info.ControllerWebhookURL}) - fmt.Println(t.Render()) - - return nil -} - -func init() { - infoCmd.AddCommand( - infoShowCmd, - ) - - rootCmd.AddCommand(infoCmd) -} diff --git a/cmd/garm-cli/cmd/init.go b/cmd/garm-cli/cmd/init.go index acdbda5a..7ed404b8 100644 --- a/cmd/garm-cli/cmd/init.go +++ b/cmd/garm-cli/cmd/init.go @@ -16,12 +16,15 @@ package cmd import ( "fmt" + "net/url" "strings" + openapiRuntimeClient "github.com/go-openapi/runtime/client" "github.com/jedib0t/go-pretty/v6/table" "github.com/pkg/errors" "github.com/spf13/cobra" + apiClientController "github.com/cloudbase/garm/client/controller" apiClientFirstRun "github.com/cloudbase/garm/client/first_run" apiClientLogin "github.com/cloudbase/garm/client/login" "github.com/cloudbase/garm/cmd/garm-cli/common" @@ -29,6 +32,12 @@ import ( "github.com/cloudbase/garm/params" ) +var ( + callbackURL string + metadataURL string + webhookURL string +) + // initCmd represents the init command var initCmd = &cobra.Command{ Use: "init", @@ -52,10 +61,13 @@ garm-cli init --name=dev --url=https://runner.example.com --username=admin --pas } } + url := strings.TrimSuffix(loginURL, "/") if err := promptUnsetInitVariables(); err != nil { return err } + ensureDefaultEndpoints(url) + newUserReq := apiClientFirstRun.NewFirstRunParams() newUserReq.Body = params.NewUserParams{ Username: loginUserName, @@ -63,9 +75,6 @@ garm-cli init --name=dev --url=https://runner.example.com --username=admin --pas FullName: loginFullName, Email: loginEmail, } - - url := strings.TrimSuffix(loginURL, "/") - initAPIClient(url, "") response, err := apiCli.FirstRun.FirstRun(newUserReq, authToken) @@ -90,17 +99,50 @@ garm-cli init --name=dev --url=https://runner.example.com --username=admin --pas Token: token.Payload.Token, }) + authToken = openapiRuntimeClient.BearerToken(token.Payload.Token) cfg.ActiveManager = loginProfileName if err := cfg.SaveConfig(); err != nil { return errors.Wrap(err, "saving config") } - renderUserTable(response.Payload) + updateUrlsReq := apiClientController.NewUpdateControllerParams() + updateUrlsReq.Body = params.UpdateControllerParams{ + MetadataURL: &metadataURL, + CallbackURL: &callbackURL, + WebhookURL: &webhookURL, + } + + controllerInfoResponse, err := apiCli.Controller.UpdateController(updateUrlsReq, authToken) + renderResponseMessage(response.Payload, controllerInfoResponse.Payload, err) return nil }, } +func ensureDefaultEndpoints(loginURL string) (err error) { + if metadataURL == "" { + metadataURL, err = url.JoinPath(loginURL, "api/v1/metadata") + if err != nil { + return err + } + } + + if callbackURL == "" { + callbackURL, err = url.JoinPath(loginURL, "api/v1/callbacks") + if err != nil { + return err + } + } + + if webhookURL == "" { + webhookURL, err = url.JoinPath(loginURL, "webhooks") + if err != nil { + return err + } + } + return nil +} + func promptUnsetInitVariables() error { var err error if loginUserName == "" { @@ -118,11 +160,18 @@ func promptUnsetInitVariables() error { } if loginPassword == "" { - loginPassword, err = common.PromptPassword("Password") + passwd, err := common.PromptPassword("Password", "") + if err != nil { + return err + } + + _, err = common.PromptPassword("Confirm password", passwd) if err != nil { return err } + loginPassword = passwd } + return nil } @@ -133,13 +182,16 @@ func init() { initCmd.Flags().StringVarP(&loginURL, "url", "a", "", "The base URL for the runner manager API") initCmd.Flags().StringVarP(&loginUserName, "username", "u", "", "The desired administrative username") initCmd.Flags().StringVarP(&loginEmail, "email", "e", "", "Email address") + initCmd.Flags().StringVarP(&metadataURL, "metadata-url", "m", "", "The metadata URL for the controller (ie. https://garm.example.com/api/v1/metadata)") + initCmd.Flags().StringVarP(&callbackURL, "callback-url", "c", "", "The callback URL for the controller (ie. https://garm.example.com/api/v1/callbacks)") + initCmd.Flags().StringVarP(&webhookURL, "webhook-url", "w", "", "The webhook URL for the controller (ie. https://garm.example.com/webhooks)") initCmd.Flags().StringVarP(&loginFullName, "full-name", "f", "", "Full name of the user") initCmd.Flags().StringVarP(&loginPassword, "password", "p", "", "The admin password") initCmd.MarkFlagRequired("name") //nolint initCmd.MarkFlagRequired("url") //nolint } -func renderUserTable(user params.User) { +func renderUserTable(user params.User) string { t := table.NewWriter() header := table.Row{"Field", "Value"} t.AppendHeader(header) @@ -148,5 +200,54 @@ func renderUserTable(user params.User) { t.AppendRow(table.Row{"Username", user.Username}) t.AppendRow(table.Row{"Email", user.Email}) t.AppendRow(table.Row{"Enabled", user.Enabled}) - fmt.Println(t.Render()) + return t.Render() +} + +func renderResponseMessage(user params.User, controllerInfo params.ControllerInfo, err error) { + userTable := renderUserTable(user) + controllerInfoTable := renderControllerInfoTable(controllerInfo) + + headerMsg := `Congrats! Your controller is now initialized. + +Following are the details of the admin user and details about the controller. + +Admin user information: + +%s +` + + controllerMsg := `Controller information: + +%s + +Make sure that the URLs in the table above are reachable by the relevant parties. + +The metadata and callback URLs *must* be accessible by the runners that GARM spins up. +The base webhook and the controller webhook URLs must be accessible by GitHub or GHES. +` + + controllerErrorMsg := `WARNING: Failed to set the required controller URLs with error: %q + +Please run: + + garm-cli controller show + +To make sure that the callback, metadata and webhook URLs are set correctly. If not, +you must set them up by running: + + garm-cli controller update \ + --metadata-url= \ + --callback-url= \ + --webhook-url= + +See the help message for garm-cli controller update for more information. +` + var ctrlMsg string + if err != nil { + ctrlMsg = fmt.Sprintf(controllerErrorMsg, err) + } else { + ctrlMsg = fmt.Sprintf(controllerMsg, controllerInfoTable) + } + + fmt.Printf("%s\n%s\n", fmt.Sprintf(headerMsg, userTable), ctrlMsg) } diff --git a/cmd/garm-cli/cmd/profile.go b/cmd/garm-cli/cmd/profile.go index 0ec28ec0..70d5176e 100644 --- a/cmd/garm-cli/cmd/profile.go +++ b/cmd/garm-cli/cmd/profile.go @@ -264,7 +264,7 @@ func promptUnsetLoginVariables() error { } if loginPassword == "" { - loginPassword, err = common.PromptPassword("Password") + loginPassword, err = common.PromptPassword("Password", "") if err != nil { return err } diff --git a/cmd/garm-cli/common/common.go b/cmd/garm-cli/common/common.go index b3850e31..3fc6c339 100644 --- a/cmd/garm-cli/common/common.go +++ b/cmd/garm-cli/common/common.go @@ -16,12 +16,13 @@ package common import ( "errors" + "fmt" "github.com/manifoldco/promptui" "github.com/nbutton23/zxcvbn-go" ) -func PromptPassword(label string) (string, error) { +func PromptPassword(label string, compareTo string) (string, error) { if label == "" { label = "Password" } @@ -30,6 +31,9 @@ func PromptPassword(label string) (string, error) { if passwordStenght.Score < 4 { return errors.New("password is too weak") } + if compareTo != "" && compareTo != input { + return errors.New("passwords do not match") + } return nil } @@ -45,7 +49,7 @@ func PromptPassword(label string) (string, error) { return result, nil } -func PromptString(label string) (string, error) { +func PromptString(label string, a ...interface{}) (string, error) { validate := func(input string) error { if len(input) == 0 { return errors.New("empty input not allowed") @@ -54,7 +58,7 @@ func PromptString(label string) (string, error) { } prompt := promptui.Prompt{ - Label: label, + Label: fmt.Sprintf(label, a...), Validate: validate, } result, err := prompt.Run() diff --git a/cmd/garm/main.go b/cmd/garm/main.go index 526e2017..83d70326 100644 --- a/cmd/garm/main.go +++ b/cmd/garm/main.go @@ -41,6 +41,7 @@ import ( "github.com/cloudbase/garm/database" "github.com/cloudbase/garm/database/common" "github.com/cloudbase/garm/metrics" + "github.com/cloudbase/garm/params" "github.com/cloudbase/garm/runner" //nolint:typecheck runnerMetrics "github.com/cloudbase/garm/runner/metrics" garmUtil "github.com/cloudbase/garm/util" @@ -142,6 +143,38 @@ func setupLogging(ctx context.Context, logCfg config.Logging, hub *websocket.Hub slog.SetDefault(slog.New(wrapped)) } +func maybeUpdateURLsFromConfig(cfg config.Config, store common.Store) error { + info, err := store.ControllerInfo() + if err != nil { + return errors.Wrap(err, "fetching controller info") + } + + var updateParams params.UpdateControllerParams + + if info.MetadataURL == "" && cfg.Default.MetadataURL != "" { + updateParams.MetadataURL = &cfg.Default.MetadataURL + } + + if info.CallbackURL == "" && cfg.Default.CallbackURL != "" { + updateParams.CallbackURL = &cfg.Default.CallbackURL + } + + if info.WebhookURL == "" && cfg.Default.WebhookURL != "" { + updateParams.WebhookURL = &cfg.Default.WebhookURL + } + + if updateParams.MetadataURL == nil && updateParams.CallbackURL == nil && updateParams.WebhookURL == nil { + // nothing to update + return nil + } + + _, err = store.UpdateController(updateParams) + if err != nil { + return errors.Wrap(err, "updating controller info") + } + return nil +} + func main() { flag.Parse() if *version { @@ -181,6 +214,10 @@ func main() { log.Fatal(err) } + if err := maybeUpdateURLsFromConfig(*cfg, db); err != nil { + log.Fatal(err) + } + runner, err := runner.NewRunner(ctx, *cfg, db) if err != nil { log.Fatalf("failed to create controller: %+v", err) @@ -212,12 +249,17 @@ func main() { log.Fatal(err) } + urlsRequiredMiddleware, err := auth.NewUrlsRequiredMiddleware(db) + if err != nil { + log.Fatal(err) + } + metricsMiddleware, err := auth.NewMetricsMiddleware(cfg.JWTAuth) if err != nil { log.Fatal(err) } - router := routers.NewAPIRouter(controller, jwtMiddleware, initMiddleware, instanceMiddleware, cfg.Default.EnableWebhookManagement) + router := routers.NewAPIRouter(controller, jwtMiddleware, initMiddleware, urlsRequiredMiddleware, instanceMiddleware, cfg.Default.EnableWebhookManagement) // start the metrics collector if cfg.Metrics.Enable { diff --git a/config/config.go b/config/config.go index 5a6fd0a2..04d26ada 100644 --- a/config/config.go +++ b/config/config.go @@ -212,22 +212,24 @@ type Default struct { } func (d *Default) Validate() error { - if d.CallbackURL == "" { - return fmt.Errorf("missing callback_url") - } - _, err := url.Parse(d.CallbackURL) - if err != nil { - return fmt.Errorf("invalid callback_url: %w", err) + if d.CallbackURL != "" { + _, err := url.ParseRequestURI(d.CallbackURL) + if err != nil { + return fmt.Errorf("invalid callback_url: %w", err) + } } - if d.MetadataURL == "" { - return fmt.Errorf("missing metadata_url") + if d.MetadataURL != "" { + if _, err := url.ParseRequestURI(d.MetadataURL); err != nil { + return fmt.Errorf("invalid metadata_url: %w", err) + } } - if _, err := url.Parse(d.MetadataURL); err != nil { - return fmt.Errorf("invalid metadata_url: %w", err) + if d.WebhookURL != "" { + if _, err := url.ParseRequestURI(d.WebhookURL); err != nil { + return fmt.Errorf("invalid webhook_url: %w", err) + } } - return nil } diff --git a/config/config_test.go b/config/config_test.go index 37f1e23e..b94956fc 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -143,20 +143,20 @@ func TestDefaultSectionConfig(t *testing.T) { errString: "", }, { - name: "CallbackURL cannot be empty", + name: "CallbackURL must be valid if set", cfg: Default{ - CallbackURL: "", + CallbackURL: "bogus_url", MetadataURL: cfg.MetadataURL, }, - errString: "missing callback_url", + errString: "invalid callback_url", }, { - name: "MetadataURL cannot be empty", + name: "MetadataURL must be valid if set", cfg: Default{ CallbackURL: cfg.CallbackURL, - MetadataURL: "", + MetadataURL: "bogus_url", }, - errString: "missing metadata_url", + errString: "invalid metadata_url", }, } diff --git a/database/common/common.go b/database/common/common.go index 5af12780..4f0df368 100644 --- a/database/common/common.go +++ b/database/common/common.go @@ -129,6 +129,12 @@ type EntityPools interface { ListEntityInstances(ctx context.Context, entity params.GithubEntity) ([]params.Instance, error) } +type ControllerStore interface { + ControllerInfo() (params.ControllerInfo, error) + InitController() (params.ControllerInfo, error) + UpdateController(info params.UpdateControllerParams) (params.ControllerInfo, error) +} + //go:generate mockery --name=Store type Store interface { RepoStore @@ -141,7 +147,5 @@ type Store interface { EntityPools GithubEndpointStore GithubCredentialsStore - - ControllerInfo() (params.ControllerInfo, error) - InitController() (params.ControllerInfo, error) + ControllerStore } diff --git a/database/common/mocks/Store.go b/database/common/mocks/Store.go index 9310e5c4..4af6f403 100644 --- a/database/common/mocks/Store.go +++ b/database/common/mocks/Store.go @@ -1516,6 +1516,34 @@ func (_m *Store) UnlockJob(ctx context.Context, jobID int64, entityID string) er return r0 } +// UpdateController provides a mock function with given fields: info +func (_m *Store) UpdateController(info params.UpdateControllerParams) (params.ControllerInfo, error) { + ret := _m.Called(info) + + if len(ret) == 0 { + panic("no return value specified for UpdateController") + } + + var r0 params.ControllerInfo + var r1 error + if rf, ok := ret.Get(0).(func(params.UpdateControllerParams) (params.ControllerInfo, error)); ok { + return rf(info) + } + if rf, ok := ret.Get(0).(func(params.UpdateControllerParams) params.ControllerInfo); ok { + r0 = rf(info) + } else { + r0 = ret.Get(0).(params.ControllerInfo) + } + + if rf, ok := ret.Get(1).(func(params.UpdateControllerParams) error); ok { + r1 = rf(info) + } else { + r1 = ret.Error(1) + } + + return r0, r1 +} + // UpdateEnterprise provides a mock function with given fields: ctx, enterpriseID, param func (_m *Store) UpdateEnterprise(ctx context.Context, enterpriseID string, param params.UpdateEntityParams) (params.Enterprise, error) { ret := _m.Called(ctx, enterpriseID, param) diff --git a/database/sql/controller.go b/database/sql/controller.go index c4389cb1..8d6c3477 100644 --- a/database/sql/controller.go +++ b/database/sql/controller.go @@ -15,6 +15,8 @@ package sql import ( + "net/url" + "github.com/google/uuid" "github.com/pkg/errors" "gorm.io/gorm" @@ -23,6 +25,21 @@ import ( "github.com/cloudbase/garm/params" ) +func dbControllerToCommonController(dbInfo ControllerInfo) (params.ControllerInfo, error) { + url, err := url.JoinPath(dbInfo.WebhookBaseURL, dbInfo.ControllerID.String()) + if err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "joining webhook URL") + } + + return params.ControllerInfo{ + ControllerID: dbInfo.ControllerID, + MetadataURL: dbInfo.MetadataURL, + WebhookURL: dbInfo.WebhookBaseURL, + ControllerWebhookURL: url, + CallbackURL: dbInfo.CallbackURL, + }, nil +} + func (s *sqlDatabase) ControllerInfo() (params.ControllerInfo, error) { var info ControllerInfo q := s.conn.Model(&ControllerInfo{}).First(&info) @@ -32,9 +49,13 @@ func (s *sqlDatabase) ControllerInfo() (params.ControllerInfo, error) { } return params.ControllerInfo{}, errors.Wrap(q.Error, "fetching controller info") } - return params.ControllerInfo{ - ControllerID: info.ControllerID, - }, nil + + paramInfo, err := dbControllerToCommonController(info) + if err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "converting controller info") + } + + return paramInfo, nil } func (s *sqlDatabase) InitController() (params.ControllerInfo, error) { @@ -60,3 +81,41 @@ func (s *sqlDatabase) InitController() (params.ControllerInfo, error) { ControllerID: newInfo.ControllerID, }, nil } + +func (s *sqlDatabase) UpdateController(info params.UpdateControllerParams) (params.ControllerInfo, error) { + var dbInfo ControllerInfo + q := s.conn.Model(&ControllerInfo{}).First(&dbInfo) + if q.Error != nil { + if errors.Is(q.Error, gorm.ErrRecordNotFound) { + return params.ControllerInfo{}, errors.Wrap(runnerErrors.ErrNotFound, "fetching controller info") + } + return params.ControllerInfo{}, errors.Wrap(q.Error, "fetching controller info") + } + + if err := info.Validate(); err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "validating controller info") + } + + if info.MetadataURL != nil { + dbInfo.MetadataURL = *info.MetadataURL + } + + if info.CallbackURL != nil { + dbInfo.CallbackURL = *info.CallbackURL + } + + if info.WebhookURL != nil { + dbInfo.WebhookBaseURL = *info.WebhookURL + } + + q = s.conn.Save(&dbInfo) + if q.Error != nil { + return params.ControllerInfo{}, errors.Wrap(q.Error, "saving controller info") + } + + paramInfo, err := dbControllerToCommonController(dbInfo) + if err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "converting controller info") + } + return paramInfo, nil +} diff --git a/database/sql/models.go b/database/sql/models.go index 633a1b51..172170b2 100644 --- a/database/sql/models.go +++ b/database/sql/models.go @@ -207,6 +207,10 @@ type ControllerInfo struct { Base ControllerID uuid.UUID + + CallbackURL string + MetadataURL string + WebhookBaseURL string } type WorkflowJob struct { diff --git a/doc/config_default.md b/doc/config_default.md index b3ccde3f..40f27cd6 100644 --- a/doc/config_default.md +++ b/doc/config_default.md @@ -4,23 +4,6 @@ The `default` config section holds configuration options that don't need a categ ```toml [default] -# This URL is used by instances to send back status messages as they install -# the github actions runner. Status messages can be seen by querying the -# runner status in garm. -# Note: If you're using a reverse proxy in front of your garm installation, -# this URL needs to point to the address of the reverse proxy. Using TLS is -# highly encouraged. -callback_url = "https://garm.example.com/api/v1/callbacks" - -# This URL is used by instances to retrieve information they need to set themselves -# up. Access to this URL is granted using the same JWT token used to send back -# status updates. Once the instance transitions to "installed" or "failed" state, -# access to both the status and metadata endpoints is disabled. -# Note: If you're using a reverse proxy in front of your garm installation, -# this URL needs to point to the address of the reverse proxy. Using TLS is -# highly encouraged. -metadata_url = "https://garm.example.com/api/v1/metadata" - # Uncomment this line if you'd like to log to a file instead of standard output. # log_file = "/tmp/runner-manager.log" diff --git a/doc/quickstart.md b/doc/quickstart.md index fac82a55..e553f425 100644 --- a/doc/quickstart.md +++ b/doc/quickstart.md @@ -3,48 +3,20 @@ - [Quick start](#quick-start) - - [The GitHub PAT Personal Access Token](#the-github-pat-personal-access-token) - [Create the config folder](#create-the-config-folder) - [The config file](#the-config-file) - [The provider section](#the-provider-section) - [Starting the service](#starting-the-service) - [Using Docker](#using-docker) - [Setting up GARM as a system service](#setting-up-garm-as-a-system-service) - - [Setting up the webhook](#setting-up-the-webhook) - [Initializing GARM](#initializing-garm) - - [Creating a gitHub endpoint Optional](#creating-a-github-endpoint-optional) + - [Setting up the webhook](#setting-up-the-webhook) + - [Creating a GitHub endpoint Optional](#creating-a-github-endpoint-optional) - [Adding credentials](#adding-credentials) - [Define a repo](#define-a-repo) - [Create a pool](#create-a-pool) - -Okay, I lied. It's not that quick. But it's not that long either. I promise. - -In this guide I'm going to take you through the entire process of setting up garm from scratch. This will include editing the config file (which will probably take the longest amount of time), fetching a proper [PAT](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) (personal access token) from GitHub, setting up the webhooks endpoint, defining your repo/org/enterprise and finally setting up a runner pool. - -For the sake of this guide, we'll assume you have access to the following setup: - -* A linux machine (ARM64 or AMD64) -* Optionally, docker/podman installed on that machine -* A public IP address or port forwarding set up on your router for port `80` or `443`. You can forward any ports, we just need to remember to use the same ports when we define the webhook in github, and the two URLs in the config file (more on that later). For the sake of this guide, I will assume you have port `80` or `443` forwarded to your machine. -* An `A` record pointing to your public IP address (optional, but recommended). Alternatively, you can use the IP address directly. I will use `garm.example.com` in this guide. If you'll be using an IP address, just replace `garm.example.com` with your IP address throughout this guide. -* All config files and data will be stored in `/etc/garm`. -* A [Personal Access Token (PAT)](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) - -Why the need to expose GARM to the internet? Well, GARM uses webhooks sent by GitHub to automatically scale runners. Whenever a new job starts, a webhook is generated letting GARM know that there is a need for a runner. GARM then spins up a new runner instance and registers it with GitHub. When the job is done, the runner instance is automatically removed. This workflow is enabled by webhooks. - -## The GitHub PAT (Personal Access Token) - -Let's start by fetching a PAT so we get that out of the way. You can use the [GitHub docs](https://docs.github.com/en/authentication/keeping-your-account-and-data-secure/creating-a-personal-access-token) to create a PAT. - -For a `classic` PAT, GARM needs the following permissions to function properly (depending on the hierarchy level you want to manage): - -* ```public_repo``` - for access to a repository -* ```repo``` - for access to a private repository -* ```admin:org``` - if you plan on using this with an organization to which you have access -* ```manage_runners:enterprise``` - if you plan to use garm at the enterprise level -* ```admin:repo_hook``` - if you want to allow GARM to install webhooks on repositories * ```admin:org_hook``` - if you want to allow GARM to install webhooks on organizations This doc will be updated at a future date with the exact permissions needed in case you want to use a fine grained PAT. @@ -67,10 +39,6 @@ Open `/etc/garm/config.toml` in your favorite editor and paste the following: ```toml [default] -callback_url = "https://garm.example.com/api/v1/callbacks" -metadata_url = "https://garm.example.com/api/v1/metadata" -# This is important for webhook management. -webhook_url = "https://garm.example.com/webhooks" enable_webhook_management = true [logging] @@ -110,7 +78,6 @@ This is a minimal config, with no providers defined. In this example we have the In this sample config we: -* define the callback, webhooks and the metadata URLs * set up logging prefrences * enable metrics with authentication * set a JWT secret which is used to sign JWT tokens @@ -118,15 +85,6 @@ In this sample config we: * enable the API server on port `80` and bind it to all interfaces * set the database backend to `sqlite3` and set a passphrase for sealing secrets (just webhook secrets for now) -The callback URLs are really important and need to point back to garm. You will notice that the domain name used in these options, is the same one we defined at the beginning of this guide. If you won't use a domain name, replace `garm.example.com` with your IP address and port number. - -We need to tell garm by which addresses it can be reached. There are many ways by which GARMs API endpoints can be exposed, and there is no sane way in which GARM itself can determine if it's behind a reverse proxy or not. The metadata URL may be served by a reverse proxy with a completely different domain name than the callback URL. Both domains pointing to the same installation of GARM in the end. - -The information in these two options is used by the instances we spin up to phone home their status and to fetch the needed metadata to finish setting themselves up. For now, the metadata URL is only used to fetch the runner registration token. - -The webhook URL is used by GARM itself to know how to set up the webhooks in GitHub. Each controller will have a unique ID and GARM will use the value in `webhook_url` as a base. It will appen -We won't go too much into detail about each of the options here. Have a look at the different config sections and their respective docs for more information. - At this point, we have a valid config file, but we still need to add the `provider` section. ## The provider section @@ -147,9 +105,13 @@ All currently available providers are `external`. The easiest provider to set up is probably the LXD or Incus provider. Incus is a fork of LXD so the functionality is identical (for now). For the purpose of this document, we'll continue with LXD. You don't need an account on an external cloud. You can just use your machine. -You will need to have LXD installed and configured. There is an excellent [getting started guide](https://documentation.ubuntu.com/lxd/en/latest/getting_started/) for LXD. Follow the unix socket so no further configuration will be needed. +You will need to have LXD installed and configured. There is an excellent [getting started guide](https://documentation.ubuntu.com/lxd/en/latest/getting_started/) for LXD. Follow the instructions there to install and configure LXD, then come back here. + +Once you have LXD installed and configured, you can add the provider section to your config file. If you're connecting to the `local` LXD installation, the [config snippet for the LXD provider](https://github.com/cloudbase/garm-provider-lxd/blob/4ee4e6fc579da4a292f40e0f7deca1e396e223d0/testdata/garm-provider-lxd.toml) will work out of the box. We'll be connecting using the unix socket so no further configuration will be needed. -Go ahead and create a new config somwhere where GARM can access it and paste that entire snippet. For the purposes of this doc, we'll assume you created a new file called `/etc/garm/garm-provider-lxd.toml`. Now we need to define the external provider config in `/etc/garm/config.toml`: +Go ahead and create a new config somwhere where GARM can access it and paste that entire snippet. For the purposes of this doc, we'll assume you created a new file called `/etc/garm/garm-provider-lxd.toml`. That config file will be used by the provider itself. Remember, the providers are external executables that are called by GARM. They may have their own configs. + +We now need to define the provider in the GARM config file and tell GARM how it can find both the provider binary and the provider specific config file. To do that, open the GARM config file `/etc/garm/config.toml` in your favorite editor and paste the following config snippet at the end: ```toml [[provider]] @@ -161,6 +123,8 @@ Go ahead and create a new config somwhere where GARM can access it and paste tha config_file = "/etc/garm/garm-provider-lxd.toml" ``` +This config snippet assumes that the LXD provider executable is available, or is going to be available in `/opt/garm/providers.d/garm-provider-lxd`. If you're using the container image, the executable is already there. If you're installing GARM as a systemd service, don't worry, instructions on how to get the LXD provider executable are coming up. + ## Starting the service You can start GARM using docker or directly on your system. I'll show you both ways. @@ -260,25 +224,15 @@ ubuntu@garm:~$ sudo journalctl -u garm Check that you can make a request to the API: ```bash -ubuntu@garm:~$ curl http://garm.example.com/webhooks ubuntu@garm:~$ docker logs garm signal.NotifyContext(context.Background, [interrupt terminated]) 2023/07/17 22:21:33 Loading provider lxd_local 2023/07/17 22:21:33 registering prometheus metrics collectors 2023/07/17 22:21:33 setting up metric routes 2023/07/17 22:21:35 ignoring unknown event -172.17.0.1 - - [17/Jul/2023:22:21:35 +0000] "GET /webhooks HTTP/1.1" 200 0 "" "curl/7.81.0" ``` -Excellent! We have a working GARM installation. Now we need to set up the webhook in GitHub. - -## Setting up the webhook - -There are two options when it comes to setting up the webhook in GitHub. You can manually set up the webhook in the GitHub UI, and then use the resulting secret when creating the entity (repo, org, enterprise), or you can let GARM do it automatically if the app or PAT you're using has the [required privileges](./github_credentials.md). - -If you want to manually set up the webhooks, have a look at the [webhooks doc](./webhooks.md) for more information. - -In this guide, I'll show you how to do it automatically when adding a new repo, assuming you have the required privileges. Note, you'll still have to manually set up webhooks if you want to use GARM at the enterprise level. Automatic webhook management is only available for repos and orgs. +Excellent! We have a working GARM installation. Now we need to initialize the controller and set up the webhook in GitHub. ## Initializing GARM @@ -293,29 +247,42 @@ wget -q -O - https://github.com/cloudbase/garm/releases/download/v0.1.4/garm-cli Now we can initialize GARM: ```bash -ubuntu@garm:~$ garm-cli init --name="local_garm" --url https://garm.example.com +ubuntu@garm:~$ garm-cli init --name="local_garm" --url http://garm.example.com Username: admin -Email: root@localhost -✔ Password: ************* +Email: admin@garm.example.com +✔ Password: ************█ +✔ Confirm password: ************█ +Congrats! Your controller is now initialized. + +Following are the details of the admin user and details about the controller. + +Admin user information: + +----------+--------------------------------------+ | FIELD | VALUE | +----------+--------------------------------------+ -| ID | ef4ab6fd-1252-4d5a-ba5a-8e8bd01610ae | +| ID | 6b0d8f67-4306-4702-80b6-eb0e2e4ee695 | | Username | admin | -| Email | root@localhost | +| Email | admin@garm.example.com | | Enabled | true | +----------+--------------------------------------+ -``` -The init command also created a local CLI profile for your new GARM server: +Controller information: -```bash -ubuntu@garm:~$ garm-cli profile list -+----------------------+--------------------------+ -| NAME | BASE URL | -+----------------------+--------------------------+ -| local_garm (current) | https://garm.example.com | -+----------------------+--------------------------+ ++------------------------+-----------------------------------------------------------------------+ +| FIELD | VALUE | ++------------------------+-----------------------------------------------------------------------+ +| Controller ID | 0c54fd66-b78b-450a-b41a-65af2fd0f71b | +| Metadata URL | http://garm.example.com/api/v1/metadata | +| Callback URL | http://garm.example.com/api/v1/callbacks | +| Webhook Base URL | http://garm.example.com/webhooks | +| Controller Webhook URL | http://garm.example.com/webhooks/0c54fd66-b78b-450a-b41a-65af2fd0f71b | ++------------------------+-----------------------------------------------------------------------+ + +Make sure that the URLs in the table above are reachable by the relevant parties. + +The metadata and callback URLs *must* be accessible by the runners that GARM spins up. +The base webhook and the controller webhook URLs must be accessible by GitHub or GHES. ``` Every time you init a new GARM instance, a new profile will be created in your local `garm-cli` config. You can also log into an already initialized instance using: @@ -332,7 +299,15 @@ Then you can switch between profiles using: garm-cli profile switch another_garm ``` -## Creating a gitHub endpoint (Optional) +## Setting up the webhook + +There are two options when it comes to setting up the webhook in GitHub. You can manually set up the webhook in the GitHub UI, and then use the resulting secret when creating the entity (repo, org, enterprise), or you can let GARM do it automatically if the app or PAT you're using has the [required privileges](./github_credentials.md). + +If you want to manually set up the webhooks, have a look at the [webhooks doc](./webhooks.md) for more information. + +In this guide, I'll show you how to do it automatically when adding a new repo, assuming you have the required privileges. Note, you'll still have to manually set up webhooks if you want to use GARM at the enterprise level. Automatic webhook management is only available for repos and orgs. + +## Creating a GitHub endpoint (Optional) This section is only of interest if you're using a GitHub Enterprise Server (GHES) deployment. If you're using [github.com](https://github.com), you can skip this section. diff --git a/doc/using_garm.md b/doc/using_garm.md index 564a8bf8..6880330c 100644 --- a/doc/using_garm.md +++ b/doc/using_garm.md @@ -7,34 +7,55 @@ While using the GARM cli, you will most likely spend most of your time listing p - [Using GARM](#using-garm) - - [Listing controller info](#listing-controller-info) - - [Listing configured providers](#listing-configured-providers) - - [Listing github credentials](#listing-github-credentials) - - [Adding a new repository](#adding-a-new-repository) - - [Managing repository webhooks](#managing-repository-webhooks) - - [Listing repositories](#listing-repositories) - - [Removing a repository](#removing-a-repository) - - [Adding a new organization](#adding-a-new-organization) - - [Adding an enterprise](#adding-an-enterprise) - - [Creating a runner pool](#creating-a-runner-pool) - - [Listing pools](#listing-pools) - - [Showing pool info](#showing-pool-info) - - [Deleting a pool](#deleting-a-pool) - - [Update a pool](#update-a-pool) - - [Listing runners](#listing-runners) - - [Showing runner info](#showing-runner-info) - - [Deleting a runner](#deleting-a-runner) + - [Controller operations](#controller-operations) + - [Listing controller info](#listing-controller-info) + - [Updating controller settings](#updating-controller-settings) + - [Providers](#providers) + - [Listing configured providers](#listing-configured-providers) + - [Github Endpoints](#github-endpoints) + - [Creating a GitHub Endpoint](#creating-a-github-endpoint) + - [Listing GitHub Endpoints](#listing-github-endpoints) + - [Getting information about an endpoint](#getting-information-about-an-endpoint) + - [Deleting a GitHub Endpoint](#deleting-a-github-endpoint) + - [GitHub credentials](#github-credentials) + - [Adding GitHub credentials](#adding-github-credentials) + - [Listing GitHub credentials](#listing-github-credentials) + - [Getting detailed information about credentials](#getting-detailed-information-about-credentials) + - [Deleting GitHub credentials](#deleting-github-credentials) + - [Repositories](#repositories) + - [Adding a new repository](#adding-a-new-repository) + - [Listing repositories](#listing-repositories) + - [Removing a repository](#removing-a-repository) + - [Organizations](#organizations) + - [Adding a new organization](#adding-a-new-organization) + - [Enterprises](#enterprises) + - [Adding an enterprise](#adding-an-enterprise) + - [Managing webhooks](#managing-webhooks) + - [Pools](#pools) + - [Creating a runner pool](#creating-a-runner-pool) + - [Listing pools](#listing-pools) + - [Showing pool info](#showing-pool-info) + - [Deleting a pool](#deleting-a-pool) + - [Update a pool](#update-a-pool) + - [Runners](#runners) + - [Listing runners](#listing-runners) + - [Showing runner info](#showing-runner-info) + - [Deleting a runner](#deleting-a-runner) - [The debug-log command](#the-debug-log-command) - [Listing recorded jobs](#listing-recorded-jobs) -## Listing controller info +## Controller operations + +The `controller` is essentially GARM itself. Every deployment of GARM will have its own controller ID which will be used to tag runners in github. The controller is responsible for managing runners, webhooks, repositories, organizations and enterprises. There are a few settings at the controller level which you can tweak and we will cover them below. + +### Listing controller info You can list the controller info by running the following command: ```bash -garm-cli controller-info show +garm-cli controller show +------------------------+----------------------------------------------------------------------------+ | FIELD | VALUE | +------------------------+----------------------------------------------------------------------------+ @@ -51,17 +72,46 @@ There are several things of interest in this output. * `Controller ID` - This is the unique identifier of the controller. Each GARM installation, on first run will automatically generate a unique controller ID. This is important for several reasons. For one, it allows us to run several GARM controllers on the same repos/orgs/enterprises, without accidentally clasing with each other. Each runner started by a GARM controller, will be tagged with this controller ID in order to easily identify runners that we manage. * `Hostname` - This is the hostname of the machine where GARM is running. This is purely informative. -* `Metadata URL` - This URL is configured by the user in the GARM config file, and is the URL that is presented to the runners via userdata when they get set up. Runners will connect to this URL and retrieve information they might need to set themselves up. GARM cannot automatically determine this URL, as it is dependent on the user's network setup. GARM may be hidden behind a load balancer or a reverse proxy, in which case, the URL by which the GARM controller can be accessed may be different than the IP addresses that are locally visible to GARM. -* `Callback URL` - This URL is configured by the user in the GARM config file, and is the URL that is presented to the runners via userdata when they get set up. Runners will connect to this URL and send status updates and system information (OS version, OS name, github runner agent ID, etc) to the controller. -* `Webhook Base URL` - This is the base URL for webhooks. It is configured by the user in the GARM config file. This URL can be called into by GitHub itself when hooks get triggered by a workflow. GARM needs to know when a new job is started in order to schedule the createion of a new runner. Job webhooks sent to this URL will be recorded by GARM and acter upon. While you can configure this URL directly in your GitHub repo settings, it is advised to use the `Controller Webhook URL` instead, as it is unique to each controller, and allows you to potentially install multiple GARM controller inside the same repo. -* `Controller Webhook URL` - This is the URL that GitHub will call into when a webhook is triggered. This URL is unique to each GARM controller and is the preferred URL to use in order to receive webhooks from GitHub. It serves the same purpose as the `Webhook Base URL`, but is unique to each controller, allowing you to potentially install multiple GARM controllers inside the same repo. +* `Metadata URL` - This URL is configured by the user, and is the URL that is presented to the runners via userdata when they get set up. Runners will connect to this URL and retrieve information they might need to set themselves up. GARM cannot automatically determine this URL, as it is dependent on the user's network setup. GARM may be hidden behind a load balancer or a reverse proxy, in which case, the URL by which the GARM controller can be accessed may be different than the IP addresses that are locally visible to GARM. Runners must be able to connect to this URL. +* `Callback URL` - This URL is configured by the user, and is the URL that is presented to the runners via userdata when they get set up. Runners will connect to this URL and send status updates and system information (OS version, OS name, github runner agent ID, etc) to the controller. Runners must be able to connect to this URL. +* `Webhook Base URL` - This is the base URL for webhooks. It is configured by the user in the GARM config file. This URL can be called into by GitHub itself when hooks get triggered by a workflow. GARM needs to know when a new job is started in order to schedule the createion of a new runner. Job webhooks sent to this URL will be recorded by GARM and acter upon. While you can configure this URL directly in your GitHub repo settings, it is advised to use the `Controller Webhook URL` instead, as it is unique to each controller, and allows you to potentially install multiple GARM controller inside the same repo. Github must be able to connect to this URL. +* `Controller Webhook URL` - This is the URL that GitHub will call into when a webhook is triggered. This URL is unique to each GARM controller and is the preferred URL to use in order to receive webhooks from GitHub. It serves the same purpose as the `Webhook Base URL`, but is unique to each controller, allowing you to potentially install multiple GARM controllers inside the same repo. Github must be able to connect to this URL. We will see the `Controller Webhook URL` later when we set up the GitHub repo to send webhooks to GARM. -## Listing configured providers +### Updating controller settings + +Like we've mentioned before, there are 3 URLs that are very important for normal operations: + +* `metadata_url` - Must be reachable by runners +* `callback_url` - Must be reachable by runners +* `webhook_url` - Must be reachable by GitHub + +These URLs depend heavily on how GARM was set up and what the network topology of the user is set up. GARM may be behind a NAT or reverse proxy. There may be different hostnames/URL paths set up for each of the above, etc. The short of it is that we cannot determine these URLs reliably and we must ask the user to tell GARM what they are. + +We can assume that the URL that the user logs in at to manage garm is the same URL that the rest of the URLs are present at, but that is just an assumption. By default, when you initialize GARM for the first time, we make this assumption to make things easy. It's also safe to assume that most users will do this anyway, but in case you don't, you will need to update the URLs in the controller and tell GARM what they are. + +In the previous section we saw that most URLs were set to `https://garm.example.com`. The URL path was the same as the routes that GARM sets up. For example, the `metadata_url` has `/api/v1/metadata`. The `callback_url` has `/api/v1/callbacks` and the `webhook_url` has `/webhooks`. This is the default setup and is what most users will use. + +If you need to update these URLs, you can use the following command: + +```bash +garm-cli controller update \ + --metadata-url https://garm.example.com/api/v1/metadata \ + --callback-url https://garm.example.com/api/v1/callbacks \ + --webhook-url https://garm.example.com/webhooks +``` + +The `Controller Webhook URL` you saw in the previous section is automatically calculated by GARM and is essentially the `webhook_url` with the controller ID appended to it. This URL is unique to each controller and is the preferred URL to use in order to receive webhooks from GitHub. + +After updating the URLs, make sure that they are properly routed to the appropriate API endpoint in GARM **and** that they are accessible by the interested parties (runners or github). + +## Providers GARM uses providers to create runners. These providers are external executables that GARM calls into to create runners in a particular IaaS. +### Listing configured providers + Once configured (see [provider configuration](/doc/providers.md)), you can list the configured providers by running the following command: ```bash @@ -87,124 +137,237 @@ ubuntu@garm:~$ garm-cli provider list Each of these providers can be used to set up a runner pool for a repository, organization or enterprise. -## Listing github credentials +## Github Endpoints -GARM needs access to your GitHub repositories, organizations or enterprise in order to manage runners. This is done via a [GitHub personal access token](/doc/github_credentials.md). You can configure multiple tokens with access to various repositories, organizations or enterprises, either on GitHub or on GitHub Enterprise Server. +GARM can be used to manage runners for repos, orgs and enterprises hosted on `github.com` or on a GitHub Enterprise Server. -The credentials sections allow you to override the API URL, Upload URL and base URLs, unlocking the ability to use GARM with GitHub Enterprise Server. +Endpoints are the way that GARM identifies where the credentials and entities you create, are located and where the API endpoints for the GitHub API can be reached, along with a possible CA certificate that validates the connection. There is a default endpoint for `github.com`, so you don't need to add it. But if you're using GHES, you'll need to add an endpoint for it. -To list existing credentials, run the following command: +### Creating a GitHub Endpoint + +To create a GitHub endpoint, you can run the following command: ```bash -ubuntu@garm:~$ garm-cli credentials list -+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+ -| NAME | DESCRIPTION | BASE URL | API URL | UPLOAD URL | -+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+ -| gabriel | github token or user gabriel | https://github.com | https://api.github.com/ | https://uploads.github.com/ | -+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+ -| gabriel_org | github token with org level access | https://github.com | https://api.github.com/ | https://uploads.github.com/ | -+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+ +garm-cli github endpoint create \ + --base-url https://ghes.example.com \ + --api-base-url https://api.ghes.example.com \ + --upload-url https://upload.ghes.example.com \ + --ca-cert-path $HOME/ca-cert.pem \ + --name example \ + --description "Just an example ghes endpoint" ++----------------+------------------------------------------------------------------+ +| FIELD | VALUE | ++----------------+------------------------------------------------------------------+ +| Name | example | +| Base URL | https://ghes.example.com | +| Upload URL | https://upload.ghes.example.com | +| API Base URL | https://api.ghes.example.com | +| CA Cert Bundle | -----BEGIN CERTIFICATE----- | +| | MIICBzCCAY6gAwIBAgIQX7fEm3dxkTeSc+E1uTFuczAKBggqhkjOPQQDAzA2MRkw | +| | FwYDVQQKExBHQVJNIGludGVybmFsIENBMRkwFwYDVQQDExBHQVJNIGludGVybmFs | +| | IENBMB4XDTIzMDIyNTE4MzE0NloXDTMzMDIyMjE4MzE0NlowNjEZMBcGA1UEChMQ | +| | R0FSTSBpbnRlcm5hbCBDQTEZMBcGA1UEAxMQR0FSTSBpbnRlcm5hbCBDQTB2MBAG | +| | ByqGSM49AgEGBSuBBAAiA2IABKat241Jzvkl+ksDuPq5jFf9wb5/l54NbGYYfcrs | +| | 4d9/sNXtPP1y8pM61hs+hCltN9UEwtxqr48q5G7Oc3IjH/dddzJTDC2bLcpwysrC | +| | NYLGtSfNj+o/8AQMwwclAY7t4KNhMF8wDgYDVR0PAQH/BAQDAgIEMB0GA1UdJQQW | +| | MBQGCCsGAQUFBwMCBggrBgEFBQcDATAPBgNVHRMBAf8EBTADAQH/MB0GA1UdDgQW | +| | BBSY+cSG07sIU2UC+fOniODKUGqiUTAKBggqhkjOPQQDAwNnADBkAjBcFz3cZ7vO | +| | IFVzqn9eqXMmZDGp58HGneHhFhJsJtQE4BkxGQmgZJ2OgTGXDqjXG3wCMGMQRALt | +| | JxwlI1PJJj7M0g48viS4NjT4kq2t/UFIbTy78aarFynUfykpL9FD9NOmiQ== | +| | -----END CERTIFICATE----- | +| | | ++----------------+------------------------------------------------------------------+ ``` -These credentials are configured in the GARM config file. You can add, remove or modify them as needed. When using GitHub, you don't need to explicitly set the API URL, Upload URL or base URL, as they are automatically set to the GitHub defaults. When using GitHub Enterprise Server, you will need to set these URLs explicitly. See the [github credentials](/doc/github_credentials.md) section for more details. +The name of the endpoint needs to be unique within GARM. -## Adding a new repository +### Listing GitHub Endpoints -To add a new repository we need to use credentials that has access to the repository. We've listed credentials above, so let's add our first repository: +To list existing GitHub endpoints, run the following command: ```bash -ubuntu@garm:~$ garm-cli repository add \ - --name garm \ - --owner gabriel-samfira \ - --credentials gabriel \ - --install-webhook \ - --random-webhook-secret -+----------------------+--------------------------------------+ -| FIELD | VALUE | -+----------------------+--------------------------------------+ -| ID | be3a0673-56af-4395-9ebf-4521fea67567 | -| Owner | gabriel-samfira | -| Name | garm | -| Credentials | gabriel | -| Pool manager running | true | -+----------------------+--------------------------------------+ +garm-cli github endpoint list ++------------+--------------------------+-------------------------------+ +| NAME | BASE URL | DESCRIPTION | ++------------+--------------------------+-------------------------------+ +| github.com | https://github.com | The github.com endpoint | ++------------+--------------------------+-------------------------------+ +| example | https://ghes.example.com | Just an example ghes endpoint | ++------------+--------------------------+-------------------------------+ ``` -Lets break down the command a bit and explain what happened above. We added a new repository to GARM, that belogs to the user `gabriel-samfira` and is called `garm`. When using GitHub, this translates to `https://github.com/gabriel-samfira/garm`. +### Getting information about an endpoint -As part of the above command, we used the credentials called `gabriel` to authenticate to GitHub. If those credentials didn't have access to the repository, we would have received an error when adding the repo. +To get information about a specific endpoint, you can run the following command: -The other interesting bit about the above command is that we automatically added the `webhook` to the repository and generated a secure random secret to authenticate the webhooks that come in from GitHub for this new repo. Any webhook claiming to be for the `gabriel-samfira/garm` repo, will be validated against the secret that was generated. +```bash +garm-cli github endpoint show github.com ++--------------+-----------------------------+ +| FIELD | VALUE | ++--------------+-----------------------------+ +| Name | github.com | +| Base URL | https://github.com | +| Upload URL | https://uploads.github.com/ | +| API Base URL | https://api.github.com/ | ++--------------+-----------------------------+ +``` + +### Deleting a GitHub Endpoint + +You can delete an endpoint unless one of the following conditions is met: -### Managing repository webhooks +* The endpoint is the default endpoint for `github.com` +* The endpoint is in use by a repository, organization or enterprise +* There are credentials defined against the endpoint you are trying to remove -The `webhook` URL that was used, will correspond to the `Controller Webhook URL` that we saw earlier when we listed the controller info. Let's list it and see what it looks like: +To delete an endpoint, you can run the following command: ```bash -ubuntu@garm:~$ garm-cli repository webhook show be3a0673-56af-4395-9ebf-4521fea67567 -+--------------+----------------------------------------------------------------------------+ -| FIELD | VALUE | -+--------------+----------------------------------------------------------------------------+ -| ID | 460257636 | -| URL | https://garm.example.com/webhooks/a4dd5f41-8e1e-42a7-af53-c0ba5ff6b0b3 | -| Events | [workflow_job] | -| Active | true | -| Insecure SSL | false | -+--------------+----------------------------------------------------------------------------+ +garm-cli github endpoint delete example ``` -We can see that it's active, and the events to which it subscribed. +## GitHub credentials -The `--install-webhook` and `--random-webhook-secret` options are convenience options that allow you to quickly add a new repository to GARM and have it ready to receive webhooks from GitHub. If you don't want to install the webhook, you can add the repository without it, and then install it later using the `garm-cli repository webhook install` command (which we'll show in a second) or manually add it in the GitHub UI. +GARM needs access to your GitHub repositories, organizations or enterprise in order to manage runners. This is done via a [GitHub personal access token or via a GitHub App](/doc/github_credentials.md). You can configure multiple tokens or apps with access to various repositories, organizations or enterprises, either on GitHub or on GitHub Enterprise Server. -To uninstall a webhook from a repository, you can use the following command: +### Adding GitHub credentials + +There are two types of credentials: + +* PAT - Personal Access Token +* App - GitHub App + +To add each of these types of credentials requires slightly different command line arguments (obviously). I'm going to give you an example of both. + +To add a PAT, you can run the following command: ```bash -garm-cli repository webhook uninstall be3a0673-56af-4395-9ebf-4521fea67567 +garm-cli github credentials add \ + --name deleteme \ + --description "just a test" \ + --auth-type pat \ + --pat-oauth-token gh_yourTokenGoesHere \ + --endpoint github.com ``` -After which listing the webhook will show that it's inactive: +To add a GitHub App (only available for repos and orgs), you can run the following command: ```bash -ubuntu@garm:~$ garm-cli repository webhook show be3a0673-56af-4395-9ebf-4521fea67567 -Error: [GET /repositories/{repoID}/webhook][404] GetRepoWebhookInfo default {Error:Not Found Details:hook not found} +garm-cli github credentials add \ + --name deleteme-app \ + --description "just a test" \ + --endpoint github.com \ + --auth-type app \ + --app-id 1 \ + --app-installation-id 99 \ + --private-key-path /etc/garm/yiourGarmAppKey.2024-12-12.private-key.pem ``` -You can always add it back using: +Notice that in both cases we specified the github endpoint for which these credentials are valid. + +### Listing GitHub credentials + +To list existing credentials, run the following command: ```bash -ubuntu@garm:~$ garm-cli repository webhook install be3a0673-56af-4395-9ebf-4521fea67567 -+--------------+----------------------------------------------------------------------------+ -| FIELD | VALUE | -+--------------+----------------------------------------------------------------------------+ -| ID | 460258767 | -| URL | https://garm.example.com/webhooks/a4dd5f41-8e1e-42a7-af53-c0ba5ff6b0b3 | -| Events | [workflow_job] | -| Active | true | -| Insecure SSL | false | -+--------------+----------------------------------------------------------------------------+ +ubuntu@garm:~$ garm-cli github credentials ls ++----+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+------+ +| ID | NAME | DESCRIPTION | BASE URL | API URL | UPLOAD URL | TYPE | ++----+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+------+ +| 1 | gabriel | github token or user gabriel | https://github.com | https://api.github.com/ | https://uploads.github.com/ | pat | ++----+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+------+ +| 2 | gabriel_org | github token with org level access | https://github.com | https://api.github.com/ | https://uploads.github.com/ | app | ++----+-------------+------------------------------------+--------------------+-------------------------+-----------------------------+------+ ``` -To allow GARM to manage webhooks, the PAT you're using must have the `admin:repo_hook` and `admin:org_hook` scopes. Webhook management is not available for enterprises. For enterprises you will have to add the webhook manually. +For more information about credentials, see the [github credentials](/doc/github_credentials.md) section for more details. -To manually add a webhook, see the [webhooks](/doc/webhooks.md) section. +### Getting detailed information about credentials + +To get detailed information about one specific credential, you can run the following command: -## Listing repositories +```bash +garm-cli github credentials show 2 ++---------------+------------------------------------+ +| FIELD | VALUE | ++---------------+------------------------------------+ +| ID | 2 | +| Name | gabriel_org | +| Description | github token with org level access | +| Base URL | https://github.com | +| API URL | https://api.github.com/ | +| Upload URL | https://uploads.github.com/ | +| Type | app | +| Endpoint | github.com | +| | | +| Repositories | gsamfira/garm-testing | +| | | +| Organizations | gsamfira | ++---------------+------------------------------------+ +``` + +### Deleting GitHub credentials + +To delete a credential, you can run the following command: + +```bash +garm-cli github credentials delete 2 +``` + +Note, you may not delete credentials that are currently associated with a repository, organization or enterprise. You will need to first replace the credentials on the entity, and then you can delete the credentials. + +## Repositories + +### Adding a new repository + +To add a new repository we need to use credentials that has access to the repository. We've listed credentials above, so let's add our first repository: + +```bash +ubuntu@garm:~$ garm-cli repository add \ + --name garm \ + --owner gabriel-samfira \ + --credentials gabriel \ + --install-webhook \ + --pool-balancer-type roundrobin \ + --random-webhook-secret ++----------------------+--------------------------------------+ +| FIELD | VALUE | ++----------------------+--------------------------------------+ +| ID | 0c91d9fd-2417-45d4-883c-05daeeaa8272 | +| Owner | gabriel-samfira | +| Name | garm | +| Pool balancer type | roundrobin | +| Credentials | gabriel | +| Pool manager running | true | ++----------------------+--------------------------------------+ +``` + +Lets break down the command a bit and explain what happened above. We added a new repository to GARM, that belogs to the user `gabriel-samfira` and is called `garm`. When using GitHub, this translates to `https://github.com/gabriel-samfira/garm`. + +As part of the above command, we used the credentials called `gabriel` to authenticate to GitHub. If those credentials didn't have access to the repository, we would have received an error when adding the repo. + +The other interesting bit about the above command is that we automatically added the `webhook` to the repository and generated a secure random secret to authenticate the webhooks that come in from GitHub for this new repo. Any webhook claiming to be for the `gabriel-samfira/garm` repo, will be validated against the secret that was generated. + +Another important aspect to remember is that once the entity (in this case a repository) is created, the credentials associated with the repo at creation time, dictates the GitHub endpoint in which this repository exists. + +When updating credentials for this entity, the new credentials **must** be associated with the same endpoint as the old ones. An error is returned if the repo is associated with `github.com` but the new credentials you're trying to set are associated with a GHES endpoint. + +### Listing repositories To list existing repositories, run the following command: ```bash ubuntu@garm:~$ garm-cli repository list -+--------------------------------------+-----------------+---------+------------------+------------------+ -| ID | OWNER | NAME | CREDENTIALS NAME | POOL MGR RUNNING | -+--------------------------------------+-----------------+---------+------------------+------------------+ -| be3a0673-56af-4395-9ebf-4521fea67567 | gabriel-samfira | garm | gabriel | true | -+--------------------------------------+-----------------+---------+------------------+------------------+ ++--------------------------------------+-----------------+--------------+------------------+--------------------+------------------+ +| ID | OWNER | NAME | CREDENTIALS NAME | POOL BALANCER TYPE | POOL MGR RUNNING | ++--------------------------------------+-----------------+--------------+------------------+--------------------+------------------+ +| be3a0673-56af-4395-9ebf-4521fea67567 | gabriel-samfira | garm | gabriel | roundrobin | true | ++--------------------------------------+-----------------+--------------+------------------+--------------------+------------------+ ``` This will list all the repositories that GARM is currently managing. -## Removing a repository +### Removing a repository To remove a repository, you can use the following command: @@ -216,7 +379,9 @@ This will remove the repository from GARM, and if a webhook was installed, will Note: GARM will not remove a webhook that points to the `Base Webhook URL`. It will only remove webhooks that are namespaced to the running controller. -## Adding a new organization +## Organizations + +### Adding a new organization Adding a new organization is similar to adding a new repository. You need to use credentials that have access to the organization, and you can add the organization to GARM using the following command: @@ -242,7 +407,9 @@ Managing webhooks for organizations is similar to managing webhooks for reposito All the other operations that exist on repositories, like listing, removing, etc, also exist for organizations and enterprises. Have a look at the help for the `garm-cli organization` subcommand for more details. -## Adding an enterprise +## Enterprises + +### Adding an enterprise Enterprises are a bit special. Currently we don't support managing webhooks for enterprises, mainly because the level of access that would be required to do so seems a bit too much to enable in GARM itself. And considering that you'll probably ever only have one enterprise with multiple organizations and repositories, the effort/risk to benefit ratio makes this feature not worth implementing at the moment. @@ -263,7 +430,66 @@ All the other operations that exist on repositories, like listing, removing, etc At that point the enterprise will be added to GARM and you can start managing runners for it. -## Creating a runner pool +## Managing webhooks + +Webhook management is available for repositories and organizations. I'm going to show you how to manage webhooks for a repository, but the same commands apply for organizations. See `--help` for more details. + +When we added the repository in the previous section, we specified the `--install-webhook` and the `--random-webhook-secret` options. These two options automatically added a webhook to the repository and generated a random secret for it. The `webhook` URL that was used, will correspond to the `Controller Webhook URL` that we saw earlier when we listed the controller info. Let's list it and see what it looks like: + +```bash +ubuntu@garm:~$ garm-cli repository webhook show be3a0673-56af-4395-9ebf-4521fea67567 ++--------------+----------------------------------------------------------------------------+ +| FIELD | VALUE | ++--------------+----------------------------------------------------------------------------+ +| ID | 460257636 | +| URL | https://garm.example.com/webhooks/a4dd5f41-8e1e-42a7-af53-c0ba5ff6b0b3 | +| Events | [workflow_job] | +| Active | true | +| Insecure SSL | false | ++--------------+----------------------------------------------------------------------------+ +``` + +We can see that it's active, and the events to which it subscribed. + +The `--install-webhook` and `--random-webhook-secret` options are convenience options that allow you to quickly add a new repository to GARM and have it ready to receive webhooks from GitHub. As long as you configured the URLs correctly (see previous sections for details), you should see a green checkmark in the GitHub settings page, under `Webhooks`. + +If you don't want to install the webhook, you can add the repository without it, and then install it later using the `garm-cli repository webhook install` command (which we'll show in a second) or manually add it in the GitHub UI. + +To uninstall a webhook from a repository, you can use the following command: + +```bash +garm-cli repository webhook uninstall be3a0673-56af-4395-9ebf-4521fea67567 +``` + +After which listing the webhook will show that it's inactive: + +```bash +ubuntu@garm:~$ garm-cli repository webhook show be3a0673-56af-4395-9ebf-4521fea67567 +Error: [GET /repositories/{repoID}/webhook][404] GetRepoWebhookInfo default {Error:Not Found Details:hook not found} +``` + +You can always add it back using: + +```bash +ubuntu@garm:~$ garm-cli repository webhook install be3a0673-56af-4395-9ebf-4521fea67567 ++--------------+----------------------------------------------------------------------------+ +| FIELD | VALUE | ++--------------+----------------------------------------------------------------------------+ +| ID | 460258767 | +| URL | https://garm.example.com/webhooks/a4dd5f41-8e1e-42a7-af53-c0ba5ff6b0b3 | +| Events | [workflow_job] | +| Active | true | +| Insecure SSL | false | ++--------------+----------------------------------------------------------------------------+ +``` + +To allow GARM to manage webhooks, the PAT or app you're using must have the `admin:repo_hook` and `admin:org_hook` scopes (or equivalent). Webhook management is not available for enterprises. For enterprises you will have to add the webhook manually. + +To manually add a webhook, see the [webhooks](/doc/webhooks.md) section. + +## Pools + +### Creating a runner pool Now that we have a repository, organization or enterprise added to GARM, we can create a runner pool for it. A runner pool is a collection of runners of the same type, that are managed by GARM and are used to run workflows for the repository, organization or enterprise. @@ -322,7 +548,7 @@ ubuntu@garm:~$ garm-cli runner list 9daa34aa-a08a-4f29-a782-f54950d8521a +----+------+--------+---------------+---------+ ``` -## Listing pools +### Listing pools To list pools created for a repository you can run: @@ -337,7 +563,20 @@ ubuntu@garm:~$ garm-cli pool list --repo=be3a0673-56af-4395-9ebf-4521fea67567 If you want to list pools for an organization or enterprise, you can use the `--org` or `--enterprise` options respectively. -## Showing pool info +You can also list **all** pools from all configureg github entities by using the `--all` option. + +```bash +ubuntu@garm:~/garm$ garm-cli pool list --all ++--------------------------------------+---------------------------+--------------+-----------------------------------------+------------------+-------+---------+---------------+----------+ +| ID | IMAGE | FLAVOR | TAGS | BELONGS TO | LEVEL | ENABLED | RUNNER PREFIX | PRIORITY | ++--------------------------------------+---------------------------+--------------+-----------------------------------------+------------------+-------+---------+---------------+----------+ +| 8935f6a6-f20f-4220-8fa9-9075e7bd7741 | windows_2022 | c3.small.x86 | self-hosted x64 Windows windows equinix | gsamfira/scripts | repo | false | garm | 0 | ++--------------------------------------+---------------------------+--------------+-----------------------------------------+------------------+-------+---------+---------------+----------+ +| 9233b3f5-2ccf-4689-8f86-a8a0d656dbeb | runner-upstream:latest | small | self-hosted x64 Linux k8s org | gsamfira | org | false | garm | 0 | ++--------------------------------------+---------------------------+--------------+-----------------------------------------+------------------+-------+---------+---------------+----------+ +``` + +### Showing pool info You can get detailed information about a pool by running the following command: @@ -365,7 +604,7 @@ ubuntu@garm:~$ garm-cli pool show 9daa34aa-a08a-4f29-a782-f54950d8521a +--------------------------+----------------------------------------+ ``` -## Deleting a pool +### Deleting a pool In order to delete a pool, you must first make sure there are no runners in the pool. To ensure this, we can first disable the pool, to make sure no new runners are created, remove the runners or allow them to be user, then we can delete the pool. @@ -401,7 +640,7 @@ If there are no runners in the pool, you can then remove it: ubuntu@garm:~$ garm-cli pool delete 9daa34aa-a08a-4f29-a782-f54950d8521a ``` -## Update a pool +### Update a pool You can update a pool by using the `garm-cli pool update` command. Nearly every aspect of a pool can be updated after it has been created. To demonstrate the command, we can enable the pool we created earlier: @@ -429,6 +668,8 @@ ubuntu@garm:~$ garm-cli pool update 9daa34aa-a08a-4f29-a782-f54950d8521a --enabl +--------------------------+----------------------------------------+ ``` +See `garm-cli pool update --help` for a list of settings that can be changed. + Now that the pool is enabled, GARM will start creating runners for it. We can list the runners in the pool to see if any have been created: ```bash @@ -453,7 +694,9 @@ root@incus:~# incus list Awesome! This runner will be able to pick up bobs that match the labels we've set on the pool. -## Listing runners +## Runners + +### Listing runners You can list runners for a pool, for a repository, organization or enterprise, or for all of them. To list all runners, you can run: @@ -474,7 +717,7 @@ ubuntu@garm:~$ garm-cli runner list --all Have a look at the help command for the flags available to the `list` subcommand. -## Showing runner info +### Showing runner info You can get detailed information about a runner by running the following command: @@ -508,7 +751,7 @@ ubuntu@garm:~$ garm-cli runner show garm-BFrp51VoVBCO +-----------------+------------------------------------------------------------------------------------------------------+ ``` -## Deleting a runner +### Deleting a runner You can delete a runner by running the following command: diff --git a/params/requests.go b/params/requests.go index 4a842ef6..25d948ee 100644 --- a/params/requests.go +++ b/params/requests.go @@ -501,3 +501,34 @@ func (u UpdateGithubCredentialsParams) Validate() error { return nil } + +type UpdateControllerParams struct { + MetadataURL *string `json:"metadata_url,omitempty"` + CallbackURL *string `json:"callback_url,omitempty"` + WebhookURL *string `json:"webhook_url,omitempty"` +} + +func (u UpdateControllerParams) Validate() error { + if u.MetadataURL != nil { + u, err := url.Parse(*u.MetadataURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return runnerErrors.NewBadRequestError("invalid metadata_url") + } + } + + if u.CallbackURL != nil { + u, err := url.Parse(*u.CallbackURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return runnerErrors.NewBadRequestError("invalid callback_url") + } + } + + if u.WebhookURL != nil { + u, err := url.Parse(*u.WebhookURL) + if err != nil || u.Scheme == "" || u.Host == "" { + return runnerErrors.NewBadRequestError("invalid webhook_url") + } + } + + return nil +} diff --git a/runner/runner.go b/runner/runner.go index 6f37c55f..2a08ae12 100644 --- a/runner/runner.go +++ b/runner/runner.go @@ -29,7 +29,6 @@ import ( "sync" "time" - "github.com/google/uuid" "github.com/juju/clock" "github.com/juju/retry" "github.com/pkg/errors" @@ -65,7 +64,6 @@ func NewRunner(ctx context.Context, cfg config.Config, db dbCommon.Store) (*Runn } poolManagerCtrl := &poolManagerCtrl{ - controllerID: ctrlID.ControllerID.String(), config: cfg, store: db, repositories: map[string]common.PoolManager{}, @@ -78,7 +76,6 @@ func NewRunner(ctx context.Context, cfg config.Config, db dbCommon.Store) (*Runn store: db, poolManagerCtrl: poolManagerCtrl, providers: providers, - controllerID: ctrlID.ControllerID, } if err := runner.loadReposOrgsAndEnterprises(); err != nil { @@ -91,9 +88,8 @@ func NewRunner(ctx context.Context, cfg config.Config, db dbCommon.Store) (*Runn type poolManagerCtrl struct { mux sync.Mutex - controllerID string - config config.Config - store dbCommon.Store + config config.Config + store dbCommon.Store repositories map[string]common.PoolManager organizations map[string]common.PoolManager @@ -345,17 +341,17 @@ func (p *poolManagerCtrl) GetEnterprisePoolManagers() (map[string]common.PoolMan } func (p *poolManagerCtrl) getInternalConfig(_ context.Context, creds params.GithubCredentials, poolBalancerType params.PoolBalancerType) (params.Internal, error) { - var controllerWebhookURL string - if p.config.Default.WebhookURL != "" { - controllerWebhookURL = fmt.Sprintf("%s/%s", p.config.Default.WebhookURL, p.controllerID) + controllerInfo, err := p.store.ControllerInfo() + if err != nil { + return params.Internal{}, errors.Wrap(err, "fetching controller info") } return params.Internal{ - ControllerID: p.controllerID, - InstanceCallbackURL: p.config.Default.CallbackURL, - InstanceMetadataURL: p.config.Default.MetadataURL, - BaseWebhookURL: p.config.Default.WebhookURL, - ControllerWebhookURL: controllerWebhookURL, + ControllerID: controllerInfo.ControllerID.String(), + InstanceCallbackURL: controllerInfo.CallbackURL, + InstanceMetadataURL: controllerInfo.MetadataURL, + BaseWebhookURL: controllerInfo.WebhookURL, + ControllerWebhookURL: controllerInfo.ControllerWebhookURL, JWTSecret: p.config.JWTAuth.Secret, PoolBalancerType: poolBalancerType, GithubCredentialsDetails: creds, @@ -372,9 +368,23 @@ type Runner struct { poolManagerCtrl PoolManagerController providers map[string]common.Provider +} + +// UpdateController will update the controller settings. +func (r *Runner) UpdateController(ctx context.Context, param params.UpdateControllerParams) (params.ControllerInfo, error) { + if !auth.IsAdmin(ctx) { + return params.ControllerInfo{}, runnerErrors.ErrUnauthorized + } + + if err := param.Validate(); err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "validating controller update params") + } - controllerInfo params.ControllerInfo - controllerID uuid.UUID + info, err := r.store.UpdateController(param) + if err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "updating controller info") + } + return info, nil } // GetControllerInfo returns the controller id and the hostname. @@ -408,19 +418,18 @@ func (r *Runner) GetControllerInfo(ctx context.Context) (params.ControllerInfo, if err != nil { return params.ControllerInfo{}, errors.Wrap(err, "fetching hostname") } - r.controllerInfo.Hostname = hostname - var controllerWebhook string - if r.controllerID != uuid.Nil && r.config.Default.WebhookURL != "" { - controllerWebhook = fmt.Sprintf("%s/%s", r.config.Default.WebhookURL, r.controllerID.String()) - } - return params.ControllerInfo{ - ControllerID: r.controllerID, - Hostname: hostname, - MetadataURL: r.config.Default.MetadataURL, - CallbackURL: r.config.Default.CallbackURL, - WebhookURL: r.config.Default.WebhookURL, - ControllerWebhookURL: controllerWebhook, - }, nil + + info, err := r.store.ControllerInfo() + if err != nil { + return params.ControllerInfo{}, errors.Wrap(err, "fetching controller info") + } + + // This is temporary. Right now, GARM is a single-instance deployment. When we add the + // ability to scale out, the hostname field will be moved form here to a dedicated node + // object. As a single controller will be made up of multiple nodes, we will need to model + // that aspect of GARM. + info.Hostname = hostname + return info, nil } func (r *Runner) ListProviders(ctx context.Context) ([]params.Provider, error) { diff --git a/testdata/config.toml b/testdata/config.toml index 23e18685..b3e5fb1d 100644 --- a/testdata/config.toml +++ b/testdata/config.toml @@ -1,33 +1,5 @@ [default] -# This URL is used by instances to send back status messages as they install -# the github actions runner. Status messages can be seen by querying the -# runner status in garm. -# Note: If you're using a reverse proxy in front of your garm installation, -# this URL needs to point to the address of the reverse proxy. Using TLS is -# highly encouraged. -callback_url = "https://garm.example.com/api/v1/callbacks" - -# This URL is used by instances to retrieve information they need to set themselves -# up. Access to this URL is granted using the same JWT token used to send back -# status updates. Once the instance transitions to "installed" or "failed" state, -# access to both the status and metadata endpoints is disabled. -# Note: If you're using a reverse proxy in front of your garm installation, -# this URL needs to point to the address of the reverse proxy. Using TLS is -# highly encouraged. -metadata_url = "https://garm.example.com/api/v1/metadata" - -# This is the base URL where GARM will listen for webhook events from github. This -# URL can be directly configured in github to send events to. -# If GARM is allowed to manage webhooks, this URL will be used as a base to optionally -# create webhooks for repositories and organizations. To avoid clashes, the unique -# controller ID that gets generated when GARM is first installed, will be added as a suffix -# to this URL. -# -# For example, assuming that your GARM controller ID is "18225ce4-e3bd-43f0-9c85-7d7858bcc5b2" -# the webhook URL will be "https://garm.example.com/webhooks/18225ce4-e3bd-43f0-9c85-7d7858bcc5b2" -webhook_url = "https://garm.example.com/webhooks" - # This option enables GARM to manage webhooks for repositories and organizations. Set this # to false to disable the API routes that manage webhooks. # @@ -249,48 +221,3 @@ disable_jit_config = false # anything (bash, a binary, python, etc). See documentation in this repo on how to write an # external provider. provider_executable = "/etc/garm/providers.d/azure/garm-external-provider" - -# This is a list of credentials that you can define as part of the repository -# or organization definitions. They are not saved inside the database, as there -# is no Vault integration (yet). This will change in the future. -# Credentials defined here can be listed using the API. Obviously, only the name -# and descriptions are returned. -[[github]] - name = "gabriel" - description = "github token or user gabriel" - # This is the type of authentication to use. It can be "pat" or "app" - auth_type = "pat" - [github.pat] - # This is a personal token with access to the repositories and organizations - # you plan on adding to garm. The "workflow" option needs to be selected in order - # to work with repositories, and the admin:org needs to be set if you plan on - # adding an organization. - oauth2_token = "super secret token" - [github.app] - # This is the app_id of the GitHub App that you want to use to authenticate - # with the GitHub API. - # This needs to be changed - app_id = 1 - # This is the private key path of the GitHub App that you want to use to authenticate - # with the GitHub API. - # This needs to be changed - private_key_path = "/etc/garm/yourAppName.2024-03-01.private-key.pem" - # This is the installation_id of the GitHub App that you want to use to authenticate - # with the GitHub API. - # This needs to be changed - installation_id = 99 - # base_url (optional) is the URL at which your GitHub Enterprise Server can be accessed. - # If these credentials are for github.com, leave this setting blank - base_url = "https://ghe.example.com" - # api_base_url (optional) is the base URL where the GitHub Enterprise Server API can be accessed. - # Leave this blank if these credentials are for github.com. - api_base_url = "https://ghe.example.com" - # upload_base_url (optional) is the base URL where the GitHub Enterprise Server upload API can be accessed. - # Leave this blank if these credentials are for github.com, or if you don't have a separate URL - # for the upload API. - upload_base_url = "https://api.ghe.example.com" - # ca_cert_bundle (optional) is the CA certificate bundle in PEM format that will be used by the github - # client to talk to the API. This bundle will also be sent to all runners as bootstrap params. - # Use this option if you're using a self signed certificate. - # Leave this blank if you're using github.com or if your certificate is signed by a valid CA. - ca_cert_bundle = "/etc/garm/ghe.crt" \ No newline at end of file