From 4bb428dfafb4b435d1dcb4ff8fa0c972b6df6e34 Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Tue, 16 Jun 2020 08:29:08 -0400 Subject: [PATCH 1/9] Implemented resume and pause call chain. Signed-off-by: Plamen Petrov --- firecracker-control/local.go | 36 ++++++ firecracker-control/service.go | 10 ++ go.mod | 1 + go.sum | 20 +++ proto/firecracker.pb.go | 80 +++++++++++- proto/service/fccontrol/fccontrol.proto | 6 + proto/service/fccontrol/ttrpc/fccontrol.pb.go | 46 ++++++- runtime/service.go | 122 ++++++++++++++++++ 8 files changed, 314 insertions(+), 7 deletions(-) diff --git a/firecracker-control/local.go b/firecracker-control/local.go index 89dbd770b..d455cd445 100644 --- a/firecracker-control/local.go +++ b/firecracker-control/local.go @@ -463,3 +463,39 @@ func setShimOOMScore(shimPid int) error { return nil } + +// PauseVM Pauses a VM +func (s *local) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.PauseVM(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// ResumeVM Resumes a VM +func (s *local) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.ResumeVM(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} diff --git a/firecracker-control/service.go b/firecracker-control/service.go index 5c931aba5..6f53f4724 100644 --- a/firecracker-control/service.go +++ b/firecracker-control/service.go @@ -96,3 +96,13 @@ func (s *service) GetVMMetadata(ctx context.Context, req *proto.GetVMMetadataReq log.G(ctx).Debug("Getting vm metadata") return s.local.GetVMMetadata(ctx, req) } + +func (s *service) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("pause VM request: %+v", req) + return s.local.PauseVM(ctx, req) +} + +func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("resume VM request: %+v", req) + return s.local.ResumeVM(ctx, req) +} diff --git a/go.mod b/go.mod index 4ed72e0e8..fea7068b2 100644 --- a/go.mod +++ b/go.mod @@ -40,6 +40,7 @@ require ( github.com/sirupsen/logrus v1.6.0 github.com/stretchr/testify v1.6.1 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 // indirect + github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c github.com/urfave/cli v1.20.0 // indirect github.com/vishvananda/netlink v1.1.0 go.etcd.io/bbolt v1.3.1-etcd.8 // indirect diff --git a/go.sum b/go.sum index 56e869abc..2b3fd07aa 100644 --- a/go.sum +++ b/go.sum @@ -320,6 +320,8 @@ github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2 h1:b6uOv7YOFK0 github.com/syndtr/gocapability v0.0.0-20180916011248-d98352740cb2/go.mod h1:hkRG7XYTFWNJGYcbNJQlaLq0fg1yr4J4t/NcTQtrfww= github.com/tidwall/pretty v1.0.0 h1:HsD+QiTn7sK6flMKIvNmpqz1qrpP3Ps6jOKIKMooyg4= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDHS3lPnIRmfVJ5Sxy3ao2SIdysLQ= +github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf h1:3J37+NPjNyGW/dbfXtj3yWuF9OEepIdGOXRaJGbORV8= @@ -350,8 +352,15 @@ golang.org/x/crypto v0.0.0-20190530122614-20be4c3c3ed5/go.mod h1:yigFU9vqHzYiE8U golang.org/x/crypto v0.0.0-20190611184440-5c40567a22f8/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0WsgG8UdvHXe6TWjY7eL6k= golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= +golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= +golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7YapZaRvUcLFFJhusH0k= +golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= +golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= +golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -364,6 +373,7 @@ golang.org/x/net v0.0.0-20190328230028-74de082e2cca h1:hyA6yiAgbUwuWqtscNvWAI7U1 golang.org/x/net v0.0.0-20190328230028-74de082e2cca/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190404232315-eb5bcb51f2a3/go.mod h1:t9HGtf8HONx5eT2rtn7q6eTqICYqUVnKs3thJo3Qplg= golang.org/x/net v0.0.0-20190613194153-d28f0bde5980/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= +golang.org/x/net v0.0.0-20190620200207-3b0461eec859/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297 h1:k7pJ2yAPLPgbskkFdhRCsA77k2fySZ1zf2zCjvQCiIM= golang.org/x/net v0.0.0-20190827160401-ba9fcec4b297/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= golang.org/x/net v0.0.0-20200226121028-0de0cce0169b/go.mod h1:z5CRVTTTmAJ677TzLLGU+0bjPO0LkuOLi4/5GtJWs/s= @@ -377,6 +387,8 @@ golang.org/x/sync v0.0.0-20190227155943-e225da77a7e6/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20190412183630-56d357773e84/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEhagSSnXWGV915qUMm9mrU= golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= +golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -415,7 +427,15 @@ golang.org/x/tools v0.0.0-20190416151739-9c9e1878f421/go.mod h1:LCzVGOaR6xXOjkQ3 golang.org/x/tools v0.0.0-20190420181800-aa740d480789/go.mod h1:LCzVGOaR6xXOjkQ3onu1FJEFr0SW1gC7cKk1uF8kGRs= golang.org/x/tools v0.0.0-20190531172133-b3315ee88b7d/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= golang.org/x/tools v0.0.0-20190614205625-5aca471b1d59/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20190617190820-da514acc4774 h1:CQVOmarCBFzTx0kbOU0ru54Cvot8SdSrNYjZPhQl+gk= golang.org/x/tools v0.0.0-20190617190820-da514acc4774/go.mod h1:/rFqwRUd4F7ZHNgwSSTFct+R/Kf4OFW1sUzUTQQTgfc= +golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtnZ6UAqBI28+e2cm9otk0dWdXHAEo= +golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= +golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixaA66WGWmrijSlxaM+JM/g= +golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= google.golang.org/genproto v0.0.0-20181109154231-b5d43981345b h1:WkFtVmaZoTRVoRYr0LTC9SYNhlw0X0HrVPz2OVssVm4= diff --git a/proto/firecracker.pb.go b/proto/firecracker.pb.go index 2d9697e42..a662b77b2 100644 --- a/proto/firecracker.pb.go +++ b/proto/firecracker.pb.go @@ -569,6 +569,82 @@ func (m *GetVMMetadataResponse) GetMetadata() string { return "" } +type PauseVMRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *PauseVMRequest) Reset() { *m = PauseVMRequest{} } +func (m *PauseVMRequest) String() string { return proto.CompactTextString(m) } +func (*PauseVMRequest) ProtoMessage() {} +func (*PauseVMRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{9} +} +func (m *PauseVMRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_PauseVMRequest.Unmarshal(m, b) +} +func (m *PauseVMRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_PauseVMRequest.Marshal(b, m, deterministic) +} +func (m *PauseVMRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_PauseVMRequest.Merge(m, src) +} +func (m *PauseVMRequest) XXX_Size() int { + return xxx_messageInfo_PauseVMRequest.Size(m) +} +func (m *PauseVMRequest) XXX_DiscardUnknown() { + xxx_messageInfo_PauseVMRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_PauseVMRequest proto.InternalMessageInfo + +func (m *PauseVMRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +type ResumeVMRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *ResumeVMRequest) Reset() { *m = ResumeVMRequest{} } +func (m *ResumeVMRequest) String() string { return proto.CompactTextString(m) } +func (*ResumeVMRequest) ProtoMessage() {} +func (*ResumeVMRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{10} +} +func (m *ResumeVMRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_ResumeVMRequest.Unmarshal(m, b) +} +func (m *ResumeVMRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_ResumeVMRequest.Marshal(b, m, deterministic) +} +func (m *ResumeVMRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_ResumeVMRequest.Merge(m, src) +} +func (m *ResumeVMRequest) XXX_Size() int { + return xxx_messageInfo_ResumeVMRequest.Size(m) +} +func (m *ResumeVMRequest) XXX_DiscardUnknown() { + xxx_messageInfo_ResumeVMRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_ResumeVMRequest proto.InternalMessageInfo + +func (m *ResumeVMRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + type JailerConfig struct { NetNS string `protobuf:"bytes,1,opt,name=NetNS,json=netNS,proto3" json:"NetNS,omitempty"` // List of the physical numbers of the CPUs on which processes in that @@ -609,7 +685,7 @@ func (m *JailerConfig) Reset() { *m = JailerConfig{} } func (m *JailerConfig) String() string { return proto.CompactTextString(m) } func (*JailerConfig) ProtoMessage() {} func (*JailerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_a73317e9fb8da571, []int{9} + return fileDescriptor_a73317e9fb8da571, []int{11} } func (m *JailerConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JailerConfig.Unmarshal(m, b) @@ -689,6 +765,8 @@ func init() { proto.RegisterType((*UpdateVMMetadataRequest)(nil), "UpdateVMMetadataRequest") proto.RegisterType((*GetVMMetadataRequest)(nil), "GetVMMetadataRequest") proto.RegisterType((*GetVMMetadataResponse)(nil), "GetVMMetadataResponse") + proto.RegisterType((*PauseVMRequest)(nil), "PauseVMRequest") + proto.RegisterType((*ResumeVMRequest)(nil), "ResumeVMRequest") proto.RegisterType((*JailerConfig)(nil), "JailerConfig") } diff --git a/proto/service/fccontrol/fccontrol.proto b/proto/service/fccontrol/fccontrol.proto index a55ff09f6..c79e59cbc 100644 --- a/proto/service/fccontrol/fccontrol.proto +++ b/proto/service/fccontrol/fccontrol.proto @@ -24,4 +24,10 @@ service Firecracker { // Get Vm's instance metadata rpc GetVMMetadata(GetVMMetadataRequest) returns (GetVMMetadataResponse); + + // Pauses a VM + rpc PauseVM(PauseVMRequest) returns (google.protobuf.Empty); + + // Resumes a VM + rpc ResumeVM(ResumeVMRequest) returns (google.protobuf.Empty); } diff --git a/proto/service/fccontrol/ttrpc/fccontrol.pb.go b/proto/service/fccontrol/ttrpc/fccontrol.pb.go index 9ffb89bb3..eab0afc8d 100644 --- a/proto/service/fccontrol/ttrpc/fccontrol.pb.go +++ b/proto/service/fccontrol/ttrpc/fccontrol.pb.go @@ -27,12 +27,12 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("fccontrol.proto", fileDescriptor_b99f53e2bf82c5ef) } var fileDescriptor_b99f53e2bf82c5ef = []byte{ - // 261 bytes of a gzipped FileDescriptorProto + // 290 bytes of a gzipped FileDescriptorProto 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4f, 0x4b, 0x4e, 0xce, 0xcf, 0x2b, 0x29, 0xca, 0xcf, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x97, 0x92, 0x4e, 0xcf, 0xcf, 0x4f, 0xcf, 0x49, 0xd5, 0x07, 0xf3, 0x92, 0x4a, 0xd3, 0xf4, 0x53, 0x73, 0x0b, 0x4a, 0x2a, 0xa1, 0x92, 0x82, 0x69, 0x99, 0x45, 0xa9, 0xc9, 0x45, 0x89, 0xc9, 0xd9, 0xa9, 0x45, 0x10, 0x21, 0xa3, - 0x57, 0x4c, 0x5c, 0xdc, 0x6e, 0x08, 0x51, 0x21, 0x7d, 0x2e, 0x0e, 0xe7, 0xa2, 0xd4, 0xc4, 0x92, + 0x2b, 0xcc, 0x5c, 0xdc, 0x6e, 0x08, 0x51, 0x21, 0x7d, 0x2e, 0x0e, 0xe7, 0xa2, 0xd4, 0xc4, 0x92, 0xd4, 0x30, 0x5f, 0x21, 0x01, 0x3d, 0x18, 0x33, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x4a, 0x10, 0x49, 0xa4, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, 0x80, 0x8b, 0x2d, 0xb8, 0x24, 0xbf, 0x20, 0xcc, 0x57, 0x88, 0x4f, 0x0f, 0xc2, 0x80, 0x29, 0x16, 0xd3, 0x83, 0xb8, 0x45, 0x0f, 0xe6, @@ -41,10 +41,12 @@ var fileDescriptor_b99f53e2bf82c5ef = []byte{ 0xbc, 0xc1, 0x20, 0x41, 0xdf, 0xd4, 0x92, 0xc4, 0x94, 0xc4, 0x92, 0x44, 0x21, 0x51, 0x3d, 0x14, 0x3e, 0x21, 0x3b, 0x5d, 0xb8, 0x04, 0x42, 0x0b, 0x52, 0xc0, 0x2e, 0x87, 0x1b, 0x21, 0xa1, 0x87, 0x2e, 0x44, 0xc8, 0x14, 0x3b, 0x2e, 0x5e, 0x77, 0x34, 0x57, 0xb8, 0x63, 0x77, 0x05, 0x9a, 0x30, - 0xc4, 0x17, 0x4e, 0xca, 0x27, 0x1e, 0xca, 0x31, 0xdc, 0x78, 0x28, 0xc7, 0xd0, 0xf0, 0x48, 0x8e, - 0xf1, 0xc4, 0x23, 0x39, 0xc6, 0x0b, 0x8f, 0xe4, 0x18, 0x1f, 0x3c, 0x92, 0x63, 0x8c, 0xe2, 0x84, - 0xc7, 0x63, 0x12, 0x1b, 0xd8, 0x52, 0x63, 0x40, 0x00, 0x00, 0x00, 0xff, 0xff, 0x0c, 0x15, 0x3e, - 0x05, 0xdb, 0x01, 0x00, 0x00, + 0xd4, 0x17, 0x46, 0x5c, 0xec, 0x01, 0x89, 0xa5, 0xc5, 0xa0, 0xb0, 0xe5, 0xd7, 0x83, 0xb2, 0x08, + 0xd9, 0x69, 0xc2, 0xc5, 0x11, 0x94, 0x5a, 0x5c, 0x9a, 0x0b, 0x89, 0x10, 0x18, 0x93, 0x80, 0x2e, + 0x27, 0xe5, 0x13, 0x0f, 0xe5, 0x18, 0x6e, 0x3c, 0x94, 0x63, 0x68, 0x78, 0x24, 0xc7, 0x78, 0xe2, + 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0x46, 0x71, 0xc2, 0x53, 0x4c, + 0x12, 0x1b, 0x58, 0x93, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x99, 0xac, 0x28, 0xd0, 0x45, 0x02, + 0x00, 0x00, } type FirecrackerService interface { @@ -54,6 +56,8 @@ type FirecrackerService interface { SetVMMetadata(ctx context.Context, req *proto1.SetVMMetadataRequest) (*empty.Empty, error) UpdateVMMetadata(ctx context.Context, req *proto1.UpdateVMMetadataRequest) (*empty.Empty, error) GetVMMetadata(ctx context.Context, req *proto1.GetVMMetadataRequest) (*proto1.GetVMMetadataResponse, error) + PauseVM(ctx context.Context, req *proto1.PauseVMRequest) (*empty.Empty, error) + ResumeVM(ctx context.Context, req *proto1.ResumeVMRequest) (*empty.Empty, error) } func RegisterFirecrackerService(srv *github_com_containerd_ttrpc.Server, svc FirecrackerService) { @@ -100,6 +104,20 @@ func RegisterFirecrackerService(srv *github_com_containerd_ttrpc.Server, svc Fir } return svc.GetVMMetadata(ctx, &req) }, + "PauseVM": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.PauseVMRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.PauseVM(ctx, &req) + }, + "ResumeVM": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.ResumeVMRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.ResumeVM(ctx, &req) + }, }) } @@ -160,3 +178,19 @@ func (c *firecrackerClient) GetVMMetadata(ctx context.Context, req *proto1.GetVM } return &resp, nil } + +func (c *firecrackerClient) PauseVM(ctx context.Context, req *proto1.PauseVMRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "PauseVM", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) ResumeVM(ctx context.Context, req *proto1.ResumeVMRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "ResumeVM", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} diff --git a/runtime/service.go b/runtime/service.go index 2b45809b5..42e28590a 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -14,11 +14,13 @@ package main import ( + "bytes" "context" "encoding/json" "fmt" "math" "net" + "net/http" "os" "runtime/debug" "strconv" @@ -61,6 +63,8 @@ import ( "github.com/firecracker-microvm/firecracker-containerd/proto" drivemount "github.com/firecracker-microvm/firecracker-containerd/proto/service/drivemount/ttrpc" fccontrolTtrpc "github.com/firecracker-microvm/firecracker-containerd/proto/service/fccontrol/ttrpc" + + "github.com/tv42/httpunix" ) func init() { @@ -144,6 +148,9 @@ type service struct { machineConfig *firecracker.Config vsockIOPortCount uint32 vsockPortMu sync.Mutex + + // httpControlClient is to send pause/resume/snapshot commands to the microVM + httpControlClient *http.Client } func shimOpts(shimCtx context.Context) (*shim.Opts, error) { @@ -579,6 +586,8 @@ func (s *service) createVM(requestCtx context.Context, request *proto.CreateVMRe return err } + s.createHTTPControlClient() + s.logger.Info("successfully started the VM") return nil } @@ -711,6 +720,55 @@ func (s *service) GetVMMetadata(requestCtx context.Context, request *proto.GetVM return &proto.GetVMMetadataResponse{Metadata: string(metadata)}, nil } +// PauseVM Pauses a VM +func (s *service) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { + pauseReq, err := formPauseReq() + if err != nil { + s.logger.WithError(err).Error("Failed to create pause vm request") + return nil, err + } + + err = s.waitVMReady() + if err != nil { + return nil, err + } + + resp, err := s.httpControlClient.Do(pauseReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send pause VM request") + s.logger.Warn(fmt.Sprintf("response from pausevm was :%s:", resp.Status)) + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to pause VM") + return nil, err + } + + return &empty.Empty{}, nil +} + +// ResumeVM Resumes a VM +func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empty.Empty, error) { + resumeReq, err := formResumeReq() + if err != nil { + s.logger.WithError(err).Error("Failed to create resume vm request") + return nil, err + } + + resp, err := s.httpControlClient.Do(resumeReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send resume VM request") + s.logger.Warn(fmt.Sprintf("response from pausevm was :%s:", resp.Status)) + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to pause VM") + return nil, err + } + + return &empty.Empty{}, nil +} + func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker.Config, error) { for _, driveMount := range req.DriveMounts { // Verify the request specified an absolute path for the source/dest of drives. @@ -1357,3 +1415,67 @@ func (s *service) monitorVMExit() { s.logger.WithError(err).Error("failed to clean up the VM") } } + +func (s *service) createHTTPControlClient() { + u := &httpunix.Transport{ + DialTimeout: 100 * time.Millisecond, + RequestTimeout: 10 * time.Second, + ResponseHeaderTimeout: 10 * time.Second, + } + u.RegisterLocation("firecracker", s.shimDir.FirecrackerSockPath()) + + t := &http.Transport{} + t.RegisterProtocol(httpunix.Scheme, u) + + var client = http.Client{ + Transport: t, + } + + s.httpControlClient = &client +} + +func formResumeReq() (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "state": "Resumed", + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PATCH", "http+unix://firecracker/vm", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formResumeReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func formPauseReq() (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "state": "Paused", + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PATCH", "http+unix://firecracker/vm", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formPauseReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} From 6856f1ba4fa23756569924561b06d296fc7d87fc Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Wed, 17 Jun 2020 08:32:07 -0400 Subject: [PATCH 2/9] Removed deprecated logging Signed-off-by: Plamen Petrov --- runtime/service.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/runtime/service.go b/runtime/service.go index 42e28590a..ee29cefe7 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -736,7 +736,6 @@ func (s *service) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empt resp, err := s.httpControlClient.Do(pauseReq) if err != nil { s.logger.WithError(err).Error("Failed to send pause VM request") - s.logger.Warn(fmt.Sprintf("response from pausevm was :%s:", resp.Status)) return nil, err } if !strings.Contains(resp.Status, "204") { @@ -758,7 +757,6 @@ func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*em resp, err := s.httpControlClient.Do(resumeReq) if err != nil { s.logger.WithError(err).Error("Failed to send resume VM request") - s.logger.Warn(fmt.Sprintf("response from pausevm was :%s:", resp.Status)) return nil, err } if !strings.Contains(resp.Status, "204") { From 58dcb449b8cff70ada2c657d021793a92b7857a6 Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Thu, 18 Jun 2020 04:21:58 -0600 Subject: [PATCH 3/9] Added support for creating and loading snapshots of VM. Notes: 1. Uses logging-only branch from ustiugov/firecracker-go-sdk 2. Firecracker logs path is hard-coded. Signed-off-by: Plamen Petrov --- firecracker-control/local.go | 57 +++++ firecracker-control/service.go | 15 ++ go.mod | 5 +- go.sum | 10 + proto/firecracker.pb.go | 151 ++++++++++- proto/firecracker.proto | 17 ++ proto/service/fccontrol/fccontrol.proto | 9 + proto/service/fccontrol/ttrpc/fccontrol.pb.go | 90 +++++-- runtime/service.go | 238 +++++++++++++++++- 9 files changed, 566 insertions(+), 26 deletions(-) diff --git a/firecracker-control/local.go b/firecracker-control/local.go index d455cd445..fc1deea76 100644 --- a/firecracker-control/local.go +++ b/firecracker-control/local.go @@ -499,3 +499,60 @@ func (s *local) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*empt return resp, nil } + +// CreateSnapshot Creates a snapshot of a VM +func (s *local) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.CreateSnapshot(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// LoadSnapshot Loads a snapshot of a VM +func (s *local) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.LoadSnapshot(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + +// Offload Shuts down a VM started through the firecracker go sdk and deletes +// the corresponding firecracker socket. All of the other resources (shim, other sockets) +// will persist. +func (s *local) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { + client, err := s.shimFirecrackerClient(ctx, req.VMID) + if err != nil { + return nil, err + } + + defer client.Close() + + resp, err := client.Offload(ctx, req) + if err != nil { + s.logger.WithError(err).Error() + return nil, err + } + + return resp, nil +} + diff --git a/firecracker-control/service.go b/firecracker-control/service.go index 6f53f4724..a447c6396 100644 --- a/firecracker-control/service.go +++ b/firecracker-control/service.go @@ -106,3 +106,18 @@ func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*em log.G(ctx).Debugf("resume VM request: %+v", req) return s.local.ResumeVM(ctx, req) } + +func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("load snapshot request: %+v", req) + return s.local.LoadSnapshot(ctx, req) +} + +func (s *service) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("create snapshot request: %+v", req) + return s.local.CreateSnapshot(ctx, req) +} + +func (s *service) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { + log.G(ctx).Debugf("offload request: %+v", req) + return s.local.Offload(ctx, req) +} diff --git a/go.mod b/go.mod index fea7068b2..099885051 100644 --- a/go.mod +++ b/go.mod @@ -1,5 +1,7 @@ module github.com/firecracker-microvm/firecracker-containerd +replace github.com/firecracker-microvm/firecracker-go-sdk => github.com/ustiugov/firecracker-go-sdk v0.20.1-0.20200625102438-8edf287b0123 + require ( github.com/Microsoft/go-winio v0.4.14 // indirect github.com/StackExchange/wmi v0.0.0-20181212234831-e0a55b97c705 // indirect @@ -17,7 +19,8 @@ require ( github.com/docker/distribution v2.7.1+incompatible // indirect github.com/docker/go-events v0.0.0-20170721190031-9461782956ad // indirect github.com/docker/go-metrics v0.0.0-20181218153428-b84716841b82 // indirect - github.com/firecracker-microvm/firecracker-go-sdk v0.21.1-0.20200811001213-ee1e7c41b7bd + github.com/docker/go-units v0.4.0 // indirect + github.com/firecracker-microvm/firecracker-go-sdk v0.0.0-00010101000000-000000000000 github.com/go-ole/go-ole v1.2.4 // indirect github.com/godbus/dbus v0.0.0-20181025153459-66d97aec3384 // indirect github.com/gofrs/uuid v3.3.0+incompatible diff --git a/go.sum b/go.sum index 2b3fd07aa..cfdb55263 100644 --- a/go.sum +++ b/go.sum @@ -324,6 +324,9 @@ github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c h1:u6SKchux2yDvFQnDH github.com/tv42/httpunix v0.0.0-20191220191345-2ba4b9c3382c/go.mod h1:hzIxponao9Kjc7aWznkXaL4U4TWaDSs8zcsY4Ka08nM= github.com/urfave/cli v1.20.0 h1:fDqGv3UG/4jbVl/QkFwEdddtEDjh/5Ov6X+0B/3bPaw= github.com/urfave/cli v1.20.0/go.mod h1:70zkFmudgCuE/ngEzBv17Jvp/497gISqfk5gWijbERA= +github.com/ustiugov/firecracker-go-sdk v0.20.1-0.20200625102438-8edf287b0123 h1:RmVkdn3XSJSVUy/yStvqVJGQ8yw93mUtJn4N+8Aytlw= +github.com/ustiugov/firecracker-go-sdk v0.20.1-0.20200625102438-8edf287b0123/go.mod h1:zyc9BrKGePpNLbQ5y2ZtdzXEfpMJeHPeFNVpyo0S1WQ= +github.com/vektah/gqlparser v1.1.2/go.mod h1:1ycwN7Ij5njmMkPPAOaRFY4rET2Enx7IkVv3vaXspKw= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf h1:3J37+NPjNyGW/dbfXtj3yWuF9OEepIdGOXRaJGbORV8= github.com/vishvananda/netlink v0.0.0-20181108222139-023a6dafdcdf/go.mod h1:+SR5DhBJrl6ZM7CoCKvpw5BKroDKQ+PJqOg65H/2ktk= github.com/vishvananda/netlink v1.1.0 h1:1iyaYNBLmP6L0220aDnYQpo1QEV4t4hJ+xEEhhJH8j0= @@ -354,6 +357,8 @@ golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56 h1:ZpKuNIejY8P0ExLOVyKhb0 golang.org/x/crypto v0.0.0-20190617133340-57b3e21c3d56/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550 h1:ObdrDkeb4kJdCP557AjRjq69pTHfNouLtWZG7j9rPN8= golang.org/x/crypto v0.0.0-20191011191535-87dc89f01550/go.mod h1:yigFU9vqHzYiE8UmvKecakEJjdnWj3jj499lnFckfCI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9 h1:psW17arqaxU48Z5kZ0CQnkZWQJsqcURM6tKiBApRjXI= +golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/lint v0.0.0-20180702182130-06c8688daad7/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3 h1:XQyxROzUlZH+WIQwySDgnISgOivlhjIEwaQaJEJrrN0= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -361,6 +366,7 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20181005035420-146acd28ed58/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -389,6 +395,8 @@ golang.org/x/sync v0.0.0-20190423024810-112230192c58 h1:8gQV6CLnAEikrhgkHFbMAEha golang.org/x/sync v0.0.0-20190423024810-112230192c58/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e h1:vcxGaoTs7kV8m5Np9uUNQin4BrLOthgV7252N8V+FwY= golang.org/x/sync v0.0.0-20190911185100-cd5d95a43a6e/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208 h1:qwRHBd0NqMbJxfbotnDhm2ByMI1Shq4Y6oRJo21SGJA= +golang.org/x/sync v0.0.0-20200625203802-6e8e738ad208/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180928133829-e4b3c5e90611/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -433,6 +441,8 @@ golang.org/x/tools v0.0.0-20191119224855-298f0cb1881e/go.mod h1:b+2E5dAYhXwXZwtn golang.org/x/tools v0.0.0-20200130002326-2f3ba24bd6e7/go.mod h1:TB2adYChydJhpapKDTa4BR/hXlZSLoq2Wpct/0txZ28= golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixaA66WGWmrijSlxaM+JM/g= golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= +golang.org/x/tools v0.0.0-20200708183856-df98bc6d456c h1:Jt8nybBNSGn80qEV8fQLwCam6RQeX4dsxit8if67Sfc= +golang.org/x/tools v0.0.0-20200708183856-df98bc6d456c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= diff --git a/proto/firecracker.pb.go b/proto/firecracker.pb.go index a662b77b2..7f0ab3f74 100644 --- a/proto/firecracker.pb.go +++ b/proto/firecracker.pb.go @@ -645,6 +645,152 @@ func (m *ResumeVMRequest) GetVMID() string { return "" } +type CreateSnapshotRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + SnapshotFilePath string `protobuf:"bytes,2,opt,name=SnapshotFilePath,json=snapshotFilePath,proto3" json:"SnapshotFilePath,omitempty"` + MemFilePath string `protobuf:"bytes,3,opt,name=MemFilePath,json=memFilePath,proto3" json:"MemFilePath,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *CreateSnapshotRequest) Reset() { *m = CreateSnapshotRequest{} } +func (m *CreateSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*CreateSnapshotRequest) ProtoMessage() {} +func (*CreateSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{11} +} +func (m *CreateSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_CreateSnapshotRequest.Unmarshal(m, b) +} +func (m *CreateSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_CreateSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *CreateSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_CreateSnapshotRequest.Merge(m, src) +} +func (m *CreateSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_CreateSnapshotRequest.Size(m) +} +func (m *CreateSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_CreateSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_CreateSnapshotRequest proto.InternalMessageInfo + +func (m *CreateSnapshotRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +func (m *CreateSnapshotRequest) GetSnapshotFilePath() string { + if m != nil { + return m.SnapshotFilePath + } + return "" +} + +func (m *CreateSnapshotRequest) GetMemFilePath() string { + if m != nil { + return m.MemFilePath + } + return "" +} + +type LoadSnapshotRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + SnapshotFilePath string `protobuf:"bytes,2,opt,name=SnapshotFilePath,json=snapshotFilePath,proto3" json:"SnapshotFilePath,omitempty"` + MemFilePath string `protobuf:"bytes,3,opt,name=MemFilePath,json=memFilePath,proto3" json:"MemFilePath,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *LoadSnapshotRequest) Reset() { *m = LoadSnapshotRequest{} } +func (m *LoadSnapshotRequest) String() string { return proto.CompactTextString(m) } +func (*LoadSnapshotRequest) ProtoMessage() {} +func (*LoadSnapshotRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{12} +} +func (m *LoadSnapshotRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_LoadSnapshotRequest.Unmarshal(m, b) +} +func (m *LoadSnapshotRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_LoadSnapshotRequest.Marshal(b, m, deterministic) +} +func (m *LoadSnapshotRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_LoadSnapshotRequest.Merge(m, src) +} +func (m *LoadSnapshotRequest) XXX_Size() int { + return xxx_messageInfo_LoadSnapshotRequest.Size(m) +} +func (m *LoadSnapshotRequest) XXX_DiscardUnknown() { + xxx_messageInfo_LoadSnapshotRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_LoadSnapshotRequest proto.InternalMessageInfo + +func (m *LoadSnapshotRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + +func (m *LoadSnapshotRequest) GetSnapshotFilePath() string { + if m != nil { + return m.SnapshotFilePath + } + return "" +} + +func (m *LoadSnapshotRequest) GetMemFilePath() string { + if m != nil { + return m.MemFilePath + } + return "" +} + +type OffloadRequest struct { + VMID string `protobuf:"bytes,1,opt,name=VMID,json=vMID,proto3" json:"VMID,omitempty"` + XXX_NoUnkeyedLiteral struct{} `json:"-"` + XXX_unrecognized []byte `json:"-"` + XXX_sizecache int32 `json:"-"` +} + +func (m *OffloadRequest) Reset() { *m = OffloadRequest{} } +func (m *OffloadRequest) String() string { return proto.CompactTextString(m) } +func (*OffloadRequest) ProtoMessage() {} +func (*OffloadRequest) Descriptor() ([]byte, []int) { + return fileDescriptor_a73317e9fb8da571, []int{13} +} +func (m *OffloadRequest) XXX_Unmarshal(b []byte) error { + return xxx_messageInfo_OffloadRequest.Unmarshal(m, b) +} +func (m *OffloadRequest) XXX_Marshal(b []byte, deterministic bool) ([]byte, error) { + return xxx_messageInfo_OffloadRequest.Marshal(b, m, deterministic) +} +func (m *OffloadRequest) XXX_Merge(src proto.Message) { + xxx_messageInfo_OffloadRequest.Merge(m, src) +} +func (m *OffloadRequest) XXX_Size() int { + return xxx_messageInfo_OffloadRequest.Size(m) +} +func (m *OffloadRequest) XXX_DiscardUnknown() { + xxx_messageInfo_OffloadRequest.DiscardUnknown(m) +} + +var xxx_messageInfo_OffloadRequest proto.InternalMessageInfo + +func (m *OffloadRequest) GetVMID() string { + if m != nil { + return m.VMID + } + return "" +} + type JailerConfig struct { NetNS string `protobuf:"bytes,1,opt,name=NetNS,json=netNS,proto3" json:"NetNS,omitempty"` // List of the physical numbers of the CPUs on which processes in that @@ -685,7 +831,7 @@ func (m *JailerConfig) Reset() { *m = JailerConfig{} } func (m *JailerConfig) String() string { return proto.CompactTextString(m) } func (*JailerConfig) ProtoMessage() {} func (*JailerConfig) Descriptor() ([]byte, []int) { - return fileDescriptor_a73317e9fb8da571, []int{11} + return fileDescriptor_a73317e9fb8da571, []int{14} } func (m *JailerConfig) XXX_Unmarshal(b []byte) error { return xxx_messageInfo_JailerConfig.Unmarshal(m, b) @@ -767,6 +913,9 @@ func init() { proto.RegisterType((*GetVMMetadataResponse)(nil), "GetVMMetadataResponse") proto.RegisterType((*PauseVMRequest)(nil), "PauseVMRequest") proto.RegisterType((*ResumeVMRequest)(nil), "ResumeVMRequest") + proto.RegisterType((*CreateSnapshotRequest)(nil), "CreateSnapshotRequest") + proto.RegisterType((*LoadSnapshotRequest)(nil), "LoadSnapshotRequest") + proto.RegisterType((*OffloadRequest)(nil), "OffloadRequest") proto.RegisterType((*JailerConfig)(nil), "JailerConfig") } diff --git a/proto/firecracker.proto b/proto/firecracker.proto index 6a9956e0d..aa80798d5 100644 --- a/proto/firecracker.proto +++ b/proto/firecracker.proto @@ -90,6 +90,23 @@ enum DriveExposePolicy { BIND = 1; } +message CreateSnapshotRequest { + string VMID = 1; + string SnapshotFilePath = 2; + string MemFilePath = 3; +} + +message LoadSnapshotRequest { + string VMID = 1; + string SnapshotFilePath = 2; + string MemFilePath = 3; +} + +message OffloadRequest { + string VMID = 1; +} + + message JailerConfig { string NetNS = 1; // List of the physical numbers of the CPUs on which processes in that diff --git a/proto/service/fccontrol/fccontrol.proto b/proto/service/fccontrol/fccontrol.proto index c79e59cbc..5c6e085d0 100644 --- a/proto/service/fccontrol/fccontrol.proto +++ b/proto/service/fccontrol/fccontrol.proto @@ -30,4 +30,13 @@ service Firecracker { // Resumes a VM rpc ResumeVM(ResumeVMRequest) returns (google.protobuf.Empty); + + // Loads VM from snapshot + rpc LoadSnapshot(LoadSnapshotRequest) returns (google.protobuf.Empty); + + // Make a snapshot of a VM + rpc CreateSnapshot(CreateSnapshotRequest) returns (google.protobuf.Empty); + + // Offload a snapshotted VM + rpc Offload(OffloadRequest) returns (google.protobuf.Empty); } diff --git a/proto/service/fccontrol/ttrpc/fccontrol.pb.go b/proto/service/fccontrol/ttrpc/fccontrol.pb.go index eab0afc8d..f97fc42e7 100644 --- a/proto/service/fccontrol/ttrpc/fccontrol.pb.go +++ b/proto/service/fccontrol/ttrpc/fccontrol.pb.go @@ -27,26 +27,28 @@ const _ = proto.GoGoProtoPackageIsVersion3 // please upgrade the proto package func init() { proto.RegisterFile("fccontrol.proto", fileDescriptor_b99f53e2bf82c5ef) } var fileDescriptor_b99f53e2bf82c5ef = []byte{ - // 290 bytes of a gzipped FileDescriptorProto - 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0xe2, 0xe2, 0x4f, 0x4b, 0x4e, 0xce, - 0xcf, 0x2b, 0x29, 0xca, 0xcf, 0xd1, 0x2b, 0x28, 0xca, 0x2f, 0xc9, 0x97, 0x92, 0x4e, 0xcf, 0xcf, - 0x4f, 0xcf, 0x49, 0xd5, 0x07, 0xf3, 0x92, 0x4a, 0xd3, 0xf4, 0x53, 0x73, 0x0b, 0x4a, 0x2a, 0xa1, - 0x92, 0x82, 0x69, 0x99, 0x45, 0xa9, 0xc9, 0x45, 0x89, 0xc9, 0xd9, 0xa9, 0x45, 0x10, 0x21, 0xa3, - 0x2b, 0xcc, 0x5c, 0xdc, 0x6e, 0x08, 0x51, 0x21, 0x7d, 0x2e, 0x0e, 0xe7, 0xa2, 0xd4, 0xc4, 0x92, - 0xd4, 0x30, 0x5f, 0x21, 0x01, 0x3d, 0x18, 0x33, 0x28, 0xb5, 0xb0, 0x34, 0xb5, 0xb8, 0x44, 0x4a, - 0x10, 0x49, 0xa4, 0xb8, 0x20, 0x3f, 0xaf, 0x38, 0x55, 0xc8, 0x80, 0x8b, 0x2d, 0xb8, 0x24, 0xbf, - 0x20, 0xcc, 0x57, 0x88, 0x4f, 0x0f, 0xc2, 0x80, 0x29, 0x16, 0xd3, 0x83, 0xb8, 0x45, 0x0f, 0xe6, - 0x16, 0x3d, 0x57, 0x90, 0x5b, 0x84, 0x8c, 0xb8, 0x38, 0xdd, 0x53, 0x4b, 0xc2, 0x7c, 0x3d, 0xf3, - 0xd2, 0xf2, 0x85, 0x04, 0xf5, 0xe0, 0x6c, 0x98, 0x3e, 0x21, 0x64, 0x21, 0xa8, 0x2d, 0x76, 0x5c, - 0xbc, 0xc1, 0x20, 0x41, 0xdf, 0xd4, 0x92, 0xc4, 0x94, 0xc4, 0x92, 0x44, 0x21, 0x51, 0x3d, 0x14, - 0x3e, 0x21, 0x3b, 0x5d, 0xb8, 0x04, 0x42, 0x0b, 0x52, 0xc0, 0x2e, 0x87, 0x1b, 0x21, 0xa1, 0x87, - 0x2e, 0x44, 0xc8, 0x14, 0x3b, 0x2e, 0x5e, 0x77, 0x34, 0x57, 0xb8, 0x63, 0x77, 0x05, 0x9a, 0x30, - 0xd4, 0x17, 0x46, 0x5c, 0xec, 0x01, 0x89, 0xa5, 0xc5, 0xa0, 0xb0, 0xe5, 0xd7, 0x83, 0xb2, 0x08, - 0xd9, 0x69, 0xc2, 0xc5, 0x11, 0x94, 0x5a, 0x5c, 0x9a, 0x0b, 0x89, 0x10, 0x18, 0x93, 0x80, 0x2e, - 0x27, 0xe5, 0x13, 0x0f, 0xe5, 0x18, 0x6e, 0x3c, 0x94, 0x63, 0x68, 0x78, 0x24, 0xc7, 0x78, 0xe2, - 0x91, 0x1c, 0xe3, 0x85, 0x47, 0x72, 0x8c, 0x0f, 0x1e, 0xc9, 0x31, 0x46, 0x71, 0xc2, 0x53, 0x4c, - 0x12, 0x1b, 0x58, 0x93, 0x31, 0x20, 0x00, 0x00, 0xff, 0xff, 0x99, 0xac, 0x28, 0xd0, 0x45, 0x02, - 0x00, 0x00, + // 336 bytes of a gzipped FileDescriptorProto + 0x1f, 0x8b, 0x08, 0x00, 0x00, 0x00, 0x00, 0x00, 0x02, 0xff, 0x84, 0x92, 0xbb, 0x4e, 0xf3, 0x40, + 0x10, 0x85, 0xe3, 0xe2, 0xcf, 0x65, 0x7e, 0x72, 0x5b, 0x41, 0x84, 0x82, 0xe4, 0x86, 0x7e, 0x82, + 0x02, 0x25, 0x8a, 0x10, 0xb7, 0x08, 0x09, 0x0b, 0x94, 0x88, 0x14, 0x74, 0x1b, 0x7b, 0x1c, 0x10, + 0x89, 0xc7, 0xd8, 0xeb, 0x82, 0x8e, 0x92, 0x47, 0x4b, 0x49, 0x49, 0x49, 0xfc, 0x24, 0x28, 0xf1, + 0x85, 0x24, 0x42, 0xda, 0xee, 0xcc, 0x67, 0x9f, 0x99, 0xb3, 0x3b, 0x0b, 0x75, 0xd7, 0xb6, 0xd9, + 0x53, 0x01, 0x4f, 0xd1, 0x0f, 0x58, 0x71, 0xfb, 0x60, 0xc2, 0x3c, 0x99, 0x52, 0x67, 0x55, 0x8d, + 0x23, 0xb7, 0x43, 0x33, 0x5f, 0xbd, 0xa5, 0x1f, 0x9b, 0xee, 0x73, 0x40, 0x76, 0x20, 0xed, 0x17, + 0x0a, 0x12, 0xd4, 0xfd, 0xf8, 0x07, 0xff, 0xaf, 0x7f, 0xa9, 0xe8, 0x40, 0xf9, 0x22, 0x20, 0xa9, + 0x68, 0x64, 0x89, 0x06, 0x66, 0x72, 0x40, 0xaf, 0x11, 0x85, 0xaa, 0xdd, 0x5c, 0x23, 0xa1, 0xcf, + 0x5e, 0x48, 0xe2, 0x08, 0x8a, 0x43, 0xc5, 0xfe, 0xc8, 0x12, 0x35, 0x4c, 0x44, 0xf6, 0x73, 0x0b, + 0x93, 0x2c, 0x98, 0x65, 0xc1, 0xab, 0x65, 0x16, 0xd1, 0x85, 0x4a, 0x9f, 0xd4, 0xc8, 0xba, 0xf1, + 0x5c, 0x16, 0x4d, 0xcc, 0x75, 0xe6, 0x13, 0xeb, 0x28, 0x9d, 0xd2, 0x83, 0xea, 0x70, 0x09, 0x2d, + 0x52, 0xd2, 0x91, 0x4a, 0x8a, 0x3d, 0xdc, 0xa8, 0x75, 0x33, 0x2f, 0xa1, 0xf1, 0xe0, 0x3b, 0xab, + 0xe4, 0x79, 0x8b, 0x7d, 0xdc, 0x46, 0xba, 0x2e, 0x3d, 0xa8, 0xf6, 0xb7, 0x52, 0xf4, 0xff, 0x4e, + 0xb1, 0x85, 0xd3, 0x53, 0x74, 0xa1, 0x74, 0x2f, 0xa3, 0x70, 0x79, 0xb7, 0x75, 0x4c, 0x95, 0x6e, + 0xe6, 0x09, 0x94, 0x07, 0x14, 0x46, 0xb3, 0x64, 0x21, 0x99, 0xd4, 0xb9, 0x4e, 0x61, 0xe7, 0x96, + 0xa5, 0x33, 0xf4, 0xa4, 0x1f, 0x3e, 0xb1, 0x12, 0xbb, 0xb8, 0x5e, 0xea, 0xdc, 0x67, 0x50, 0x4b, + 0xf6, 0x9c, 0xfb, 0x5b, 0xb8, 0x09, 0xf4, 0x3b, 0x2e, 0xdd, 0xb9, 0xee, 0x94, 0xa5, 0x23, 0xea, + 0x98, 0x2a, 0x8d, 0xe7, 0xfc, 0x70, 0xbe, 0x30, 0x0b, 0x5f, 0x0b, 0xb3, 0xf0, 0x1e, 0x9b, 0xc6, + 0x3c, 0x36, 0x8d, 0xcf, 0xd8, 0x34, 0xbe, 0x63, 0xd3, 0x78, 0xac, 0xe4, 0xaf, 0x7c, 0x5c, 0x5c, + 0x99, 0x8e, 0x7f, 0x02, 0x00, 0x00, 0xff, 0xff, 0xfb, 0xe9, 0x16, 0xe0, 0xf9, 0x02, 0x00, 0x00, } type FirecrackerService interface { @@ -58,6 +60,9 @@ type FirecrackerService interface { GetVMMetadata(ctx context.Context, req *proto1.GetVMMetadataRequest) (*proto1.GetVMMetadataResponse, error) PauseVM(ctx context.Context, req *proto1.PauseVMRequest) (*empty.Empty, error) ResumeVM(ctx context.Context, req *proto1.ResumeVMRequest) (*empty.Empty, error) + LoadSnapshot(ctx context.Context, req *proto1.LoadSnapshotRequest) (*empty.Empty, error) + CreateSnapshot(ctx context.Context, req *proto1.CreateSnapshotRequest) (*empty.Empty, error) + Offload(ctx context.Context, req *proto1.OffloadRequest) (*empty.Empty, error) } func RegisterFirecrackerService(srv *github_com_containerd_ttrpc.Server, svc FirecrackerService) { @@ -118,6 +123,27 @@ func RegisterFirecrackerService(srv *github_com_containerd_ttrpc.Server, svc Fir } return svc.ResumeVM(ctx, &req) }, + "LoadSnapshot": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.LoadSnapshotRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.LoadSnapshot(ctx, &req) + }, + "CreateSnapshot": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.CreateSnapshotRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.CreateSnapshot(ctx, &req) + }, + "Offload": func(ctx context.Context, unmarshal func(interface{}) error) (interface{}, error) { + var req proto1.OffloadRequest + if err := unmarshal(&req); err != nil { + return nil, err + } + return svc.Offload(ctx, &req) + }, }) } @@ -194,3 +220,27 @@ func (c *firecrackerClient) ResumeVM(ctx context.Context, req *proto1.ResumeVMRe } return &resp, nil } + +func (c *firecrackerClient) LoadSnapshot(ctx context.Context, req *proto1.LoadSnapshotRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "LoadSnapshot", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) CreateSnapshot(ctx context.Context, req *proto1.CreateSnapshotRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "CreateSnapshot", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} + +func (c *firecrackerClient) Offload(ctx context.Context, req *proto1.OffloadRequest) (*empty.Empty, error) { + var resp empty.Empty + if err := c.client.Call(ctx, "Firecracker", "Offload", req, &resp); err != nil { + return nil, err + } + return &resp, nil +} diff --git a/runtime/service.go b/runtime/service.go index ee29cefe7..5c2c97d8f 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -27,6 +27,8 @@ import ( "strings" "sync" "time" + "os/exec" + "syscall" // disable gosec check for math/rand. We just need a random starting // place to start looking for CIDs; no need for cryptographically @@ -151,6 +153,8 @@ type service struct { // httpControlClient is to send pause/resume/snapshot commands to the microVM httpControlClient *http.Client + firecrackerPid int + taskDrivePathOnHost string } func shimOpts(shimCtx context.Context) (*shim.Opts, error) { @@ -485,7 +489,10 @@ func (s *service) CreateVM(requestCtx context.Context, request *proto.CreateVMRe s.logger.WithError(err).Error("failed to publish start VM event") } - go s.monitorVMExit() + // Commented out because its execution cancels the shim, and + // it would get executed on Offload if we leave it, killing the shim, + // and making snapshots impossible. + //go s.monitorVMExit() // let all the other methods know that the VM is ready for tasks close(s.vmReady) @@ -588,7 +595,14 @@ func (s *service) createVM(requestCtx context.Context, request *proto.CreateVMRe s.createHTTPControlClient() + if pid, err := s.machine.PID(); err != nil { + s.logger.WithError(err).Error("Failed to get PID of firecracker process") + return err + } else { + s.firecrackerPid = pid + } s.logger.Info("successfully started the VM") + return nil } @@ -760,7 +774,95 @@ func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*em return nil, err } if !strings.Contains(resp.Status, "204") { - s.logger.WithError(err).Error("Failed to pause VM") + s.logger.WithError(err).Error("Failed to resume VM") + return nil, err + } + return &empty.Empty{}, nil +} + +// LoadSnapshot Loads a VM from a snapshot +func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + if err := s.startFirecrackerProcess() ; err != nil { + s.logger.WithError(err).Error("startFirecrackerProcess returned an error") + return nil, err + } + time.Sleep(200 * time.Millisecond) + s.createHTTPControlClient() + + loadSnapReq, err := formLoadSnapReq(req.SnapshotFilePath, req.MemFilePath) + if err != nil { + s.logger.WithError(err).Error("Failed to create load snapshot request") + return nil, err + } + + resp, err := s.httpControlClient.Do(loadSnapReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send load snapshot request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to load VM from snapshot") + s.logger.WithError(err).Errorf("Status of request: %s", resp.Status) + return nil, err + } + + // Workaround for https://github.com/firecracker-microvm/firecracker/issues/1974 + patchDriveReq, err := formPatchDriveReq("MN2HE43UOVRDA", s.taskDrivePathOnHost) + if err != nil { + s.logger.WithError(err).Error("Failed to create patch drive request") + return nil, err + } + + resp, err = s.httpControlClient.Do(patchDriveReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send patch drive request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to patch drive") + s.logger.WithError(err).Errorf("Status of request: %s", resp.Status) + return nil, err + } + + return &empty.Empty{}, nil +} + +// CreateSnapshot Creates a snapshot of a VM +func (s *service) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotRequest) (*empty.Empty, error) { + createSnapReq, err := formCreateSnapReq(req.SnapshotFilePath, req.MemFilePath) + if err != nil { + s.logger.WithError(err).Error("Failed to create make snapshot request") + return nil, err + } + + resp, err := s.httpControlClient.Do(createSnapReq) + if err != nil { + s.logger.WithError(err).Error("Failed to send make snapshot request") + return nil, err + } + if !strings.Contains(resp.Status, "204") { + s.logger.WithError(err).Error("Failed to make snapshot of VM") + return nil, err + } + + return &empty.Empty{}, nil +} + +// Offload Shuts down a VM and deletes the corresponding firecracker socket +// and vsock. All of the other resources will persist +func (s *service) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { + if err := syscall.Kill(s.firecrackerPid, 9); err != nil { + s.logger.WithError(err).Error("Failed to kill firecracker process") + return nil, err + } + + if err := os.RemoveAll(s.shimDir.FirecrackerSockPath()); err != nil { + s.logger.WithError(err).Error("Failed to delete firecracker socket") + return nil, err + } + + if err := os.RemoveAll(s.shimDir.FirecrackerVSockPath()); err != nil { + s.logger.WithError(err).Error("Failed to delete firecracker vsock") return nil, err } @@ -786,14 +888,25 @@ func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker return nil, errors.Wrapf(err, "failed to get relative path to firecracker vsock") } + // TODO: Remove hardcoding and make a parameter + logFilePath := fmt.Sprintf("/tmp/log_%s_start.logs", s.vmID) + if err := os.RemoveAll(logFilePath); err != nil { + s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) + return nil, err + } + if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0666); err != nil { + s.logger.WithError(err).Errorf("Failed to create %s", logFilePath) + return nil, err + } + cfg := firecracker.Config{ SocketPath: relSockPath, VsockDevices: []firecracker.VsockDevice{{ Path: relVSockPath, ID: "agent_api", }}, - LogFifo: s.shimDir.FirecrackerLogFifoPath(), - MetricsFifo: s.shimDir.FirecrackerMetricsFifoPath(), + // Put LogPath insteadof LogFifo here to comply with the new Firecracker logging + LogPath: logFilePath, MachineCfg: machineConfigurationFromProto(s.config, req.MachineCfg), LogLevel: s.config.DebugHelper.GetFirecrackerLogLevel(), VMID: s.vmID, @@ -961,6 +1074,8 @@ func (s *service) Create(requestCtx context.Context, request *taskAPI.CreateTask } rootfsMnt := request.Rootfs[0] + s.taskDrivePathOnHost = rootfsMnt.Source + err = s.containerStubHandler.Reserve(requestCtx, request.ID, rootfsMnt.Source, vmBundleDir.RootfsPath(), "ext4", nil, s.driveMountClient, s.machine) if err != nil { @@ -1477,3 +1592,118 @@ func formPauseReq() (*http.Request, error) { return req, nil } + +func formLoadSnapReq(snapshotPath, memPath string) (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "snapshot_path": snapshotPath, + "mem_file_path": memPath, + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PUT", "http+unix://firecracker/snapshot/load", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formLoadSnapReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func formCreateSnapReq(snapshotPath, memPath string) (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "snapshot_type": "Full", + "snapshot_path": snapshotPath, + "mem_file_path": memPath, + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PUT", "http+unix://firecracker/snapshot/create", bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formCreateSnapReq") + return nil, err + } + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func formPatchDriveReq(drive_id, path_on_host string) (*http.Request, error) { + var req *http.Request + + data := map[string]string{ + "drive_id": drive_id, + "path_on_host": path_on_host, + } + json, err := json.Marshal(data) + if err != nil { + logrus.WithError(err).Error("Failed to marshal json data") + return nil, err + } + + req, err = http.NewRequest("PATCH", fmt.Sprintf("http+unix://firecracker/drives/%s", drive_id), bytes.NewBuffer(json)) + if err != nil { + logrus.WithError(err).Error("Failed to create new HTTP request in formPauseReq") + return nil, err + } + + req.Header.Add("accept", "application/json") + req.Header.Add("Content-Type", "application/json") + + return req, nil +} + +func (s *service) startFirecrackerProcess() error { + firecPath, err := exec.LookPath("firecracker") + if err != nil { + logrus.WithError(err).Error("failed to look up firecracker binary") + return err + } + + + // TODO: Remove hardcoding and make a parameter + logFilePath := fmt.Sprintf("/tmp/log_%s_after.logs", s.vmID) + if err := os.RemoveAll(logFilePath); err != nil { + s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) + return err + } + if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0666); err != nil { + s.logger.WithError(err).Errorf("Failed to create %s", logFilePath) + return err + } + + args := []string{ + "--api-sock", s.shimDir.FirecrackerSockPath(), + "--log-path", logFilePath, + "--level", s.config.DebugHelper.GetFirecrackerLogLevel(), + "--show-level", + "--show-log-origin", + } + + firecrackerCmd := exec.Command(firecPath, args...) + firecrackerCmd.Dir = s.shimDir.RootPath() + + if err := firecrackerCmd.Start(); err != nil { + logrus.WithError(err).Error("Failed to start firecracker process") + } + + go firecrackerCmd.Wait() + + s.firecrackerPid = firecrackerCmd.Process.Pid + + return nil +} \ No newline at end of file From c810db3c9019792ccc348aa77a654b10b8cb5a6c Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Fri, 10 Jul 2020 09:12:17 -0400 Subject: [PATCH 4/9] Altered buildVMConfiguration tests Signed-off-by: Plamen Petrov --- firecracker-control/local.go | 1 - go.mod | 1 + go.sum | 4 ++++ runtime/service.go | 46 ++++++++++++++++++++---------------- runtime/service_test.go | 10 ++++++-- 5 files changed, 38 insertions(+), 24 deletions(-) diff --git a/firecracker-control/local.go b/firecracker-control/local.go index fc1deea76..668e096fc 100644 --- a/firecracker-control/local.go +++ b/firecracker-control/local.go @@ -555,4 +555,3 @@ func (s *local) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty. return resp, nil } - diff --git a/go.mod b/go.mod index 099885051..854f41336 100644 --- a/go.mod +++ b/go.mod @@ -49,6 +49,7 @@ require ( go.etcd.io/bbolt v1.3.1-etcd.8 // indirect golang.org/x/sync v0.0.0-20190423024810-112230192c58 golang.org/x/sys v0.0.0-20200323222414-85ca7c5b95cd + golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1 // indirect google.golang.org/genproto v0.0.0-20181109154231-b5d43981345b // indirect google.golang.org/grpc v1.21.0 gotest.tools v2.2.0+incompatible // indirect diff --git a/go.sum b/go.sum index cfdb55263..0f8baf399 100644 --- a/go.sum +++ b/go.sum @@ -366,6 +366,7 @@ golang.org/x/lint v0.0.0-20200302205851-738671d3881b h1:Wh+f8QHJXR411sJR8/vRBTZ7 golang.org/x/lint v0.0.0-20200302205851-738671d3881b/go.mod h1:3xt1FjdF8hUf6vQPIChWIBhFzV8gjjsPE/fR3IyQdNY= golang.org/x/mod v0.1.1-0.20191105210325-c90efee705ee/go.mod h1:QqPTAvyqsEbceGzBzNggFXnrqF1CaUcvgkdR5Ot7KZg= golang.org/x/mod v0.2.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= +golang.org/x/mod v0.3.0 h1:RM4zey1++hCTbCVQfnWeKs9/IEsaBLA8vTkd0WVtmH4= golang.org/x/mod v0.3.0/go.mod h1:s0Qsj1ACt9ePp/hMypM3fl4fZqREWJwdYDEqhRiZZUA= golang.org/x/net v0.0.0-20180826012351-8a410e7b638d/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= golang.org/x/net v0.0.0-20180926154720-4dfa2610cdf3/go.mod h1:mL1N/T3taQHkDXs73rZJwtUhF3w3ftmwwsq0BUmARs4= @@ -443,8 +444,11 @@ golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9 h1:cwgUY+1ja2qxWb2dyaCoixa golang.org/x/tools v0.0.0-20200615222825-6aa8f57aacd9/go.mod h1:EkVYQZoAsY45+roYkvgYkIh4xh/qjgUK9TdY2XT94GE= golang.org/x/tools v0.0.0-20200708183856-df98bc6d456c h1:Jt8nybBNSGn80qEV8fQLwCam6RQeX4dsxit8if67Sfc= golang.org/x/tools v0.0.0-20200708183856-df98bc6d456c/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= +golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1 h1:rD1FcWVsRaMY+l8biE9jbWP5MS/CJJ/90a9TMkMgNrM= +golang.org/x/tools v0.0.0-20200710042808-f1c4188a97a1/go.mod h1:njjCfa9FT2d7l9Bc6FUM5FLjQPp3cFF28FI3qnDFljA= golang.org/x/xerrors v0.0.0-20190717185122-a985d3407aa7/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= golang.org/x/xerrors v0.0.0-20191011141410-1b5146add898/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= +golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543 h1:E7g+9GITq07hpfrRu66IVDexMakfv52eLZ2CXBWiKr4= golang.org/x/xerrors v0.0.0-20191204190536-9bdfabe68543/go.mod h1:I/5z698sn9Ka8TeJc9MKroUUfqBBauWjQqLJ2OPfmY0= google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM= google.golang.org/genproto v0.0.0-20180817151627-c66870c02cf8/go.mod h1:JiN7NxoALGmiZfu7CAH4rXhgtRTLTxftemlI0sWmxmc= diff --git a/runtime/service.go b/runtime/service.go index 5c2c97d8f..dde4a5e0c 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -22,13 +22,13 @@ import ( "net" "net/http" "os" + "os/exec" "runtime/debug" "strconv" "strings" "sync" - "time" - "os/exec" "syscall" + "time" // disable gosec check for math/rand. We just need a random starting // place to start looking for CIDs; no need for cryptographically @@ -152,8 +152,8 @@ type service struct { vsockPortMu sync.Mutex // httpControlClient is to send pause/resume/snapshot commands to the microVM - httpControlClient *http.Client - firecrackerPid int + httpControlClient *http.Client + firecrackerPid int taskDrivePathOnHost string } @@ -489,7 +489,7 @@ func (s *service) CreateVM(requestCtx context.Context, request *proto.CreateVMRe s.logger.WithError(err).Error("failed to publish start VM event") } - // Commented out because its execution cancels the shim, and + // Commented out because its execution cancels the shim, and // it would get executed on Offload if we leave it, killing the shim, // and making snapshots impossible. //go s.monitorVMExit() @@ -595,12 +595,14 @@ func (s *service) createVM(requestCtx context.Context, request *proto.CreateVMRe s.createHTTPControlClient() - if pid, err := s.machine.PID(); err != nil { + pid, err := s.machine.PID() + if err != nil { s.logger.WithError(err).Error("Failed to get PID of firecracker process") return err - } else { - s.firecrackerPid = pid } + + s.firecrackerPid = pid + s.logger.Info("successfully started the VM") return nil @@ -782,7 +784,7 @@ func (s *service) ResumeVM(ctx context.Context, req *proto.ResumeVMRequest) (*em // LoadSnapshot Loads a VM from a snapshot func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { - if err := s.startFirecrackerProcess() ; err != nil { + if err := s.startFirecrackerProcess(); err != nil { s.logger.WithError(err).Error("startFirecrackerProcess returned an error") return nil, err } @@ -894,7 +896,7 @@ func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) return nil, err } - if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0666); err != nil { + if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0600); err != nil { s.logger.WithError(err).Errorf("Failed to create %s", logFilePath) return nil, err } @@ -906,10 +908,10 @@ func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker ID: "agent_api", }}, // Put LogPath insteadof LogFifo here to comply with the new Firecracker logging - LogPath: logFilePath, - MachineCfg: machineConfigurationFromProto(s.config, req.MachineCfg), - LogLevel: s.config.DebugHelper.GetFirecrackerLogLevel(), - VMID: s.vmID, + LogPath: logFilePath, + MachineCfg: machineConfigurationFromProto(s.config, req.MachineCfg), + LogLevel: s.config.DebugHelper.GetFirecrackerLogLevel(), + VMID: s.vmID, } if req.JailerConfig != nil { @@ -1518,6 +1520,8 @@ func (s *service) cleanup() error { } // monitorVMExit watches the VM and cleanup resources when it terminates. +// Comment out because unused +/* func (s *service) monitorVMExit() { // Block until the VM exits if err := s.machine.Wait(s.shimCtx); err != nil && err != context.Canceled { @@ -1528,6 +1532,7 @@ func (s *service) monitorVMExit() { s.logger.WithError(err).Error("failed to clean up the VM") } } +*/ func (s *service) createHTTPControlClient() { u := &httpunix.Transport{ @@ -1642,12 +1647,12 @@ func formCreateSnapReq(snapshotPath, memPath string) (*http.Request, error) { return req, nil } -func formPatchDriveReq(drive_id, path_on_host string) (*http.Request, error) { +func formPatchDriveReq(driveID, pathOnHost string) (*http.Request, error) { var req *http.Request data := map[string]string{ - "drive_id": drive_id, - "path_on_host": path_on_host, + "drive_id": driveID, + "path_on_host": pathOnHost, } json, err := json.Marshal(data) if err != nil { @@ -1655,7 +1660,7 @@ func formPatchDriveReq(drive_id, path_on_host string) (*http.Request, error) { return nil, err } - req, err = http.NewRequest("PATCH", fmt.Sprintf("http+unix://firecracker/drives/%s", drive_id), bytes.NewBuffer(json)) + req, err = http.NewRequest("PATCH", fmt.Sprintf("http+unix://firecracker/drives/%s", driveID), bytes.NewBuffer(json)) if err != nil { logrus.WithError(err).Error("Failed to create new HTTP request in formPauseReq") return nil, err @@ -1674,14 +1679,13 @@ func (s *service) startFirecrackerProcess() error { return err } - // TODO: Remove hardcoding and make a parameter logFilePath := fmt.Sprintf("/tmp/log_%s_after.logs", s.vmID) if err := os.RemoveAll(logFilePath); err != nil { s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) return err } - if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0666); err != nil { + if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0600); err != nil { s.logger.WithError(err).Errorf("Failed to create %s", logFilePath) return err } @@ -1706,4 +1710,4 @@ func (s *service) startFirecrackerProcess() error { s.firecrackerPid = firecrackerCmd.Process.Pid return nil -} \ No newline at end of file +} diff --git a/runtime/service_test.go b/runtime/service_test.go index 7cc842a1e..80e690221 100644 --- a/runtime/service_test.go +++ b/runtime/service_test.go @@ -253,8 +253,14 @@ func TestBuildVMConfiguration(t *testing.T) { Path: relVSockPath, ID: "agent_api", }} - tc.expectedCfg.LogFifo = svc.shimDir.FirecrackerLogFifoPath() - tc.expectedCfg.MetricsFifo = svc.shimDir.FirecrackerMetricsFifoPath() + + // Remove LogFifo and MetricsInfo in order to comply + // with new Firecracker logging. + // Include hard coded LogPath + // TODO: FIX TEST WHEN LogPath is no longer hardcoded + //tc.expectedCfg.LogFifo = svc.shimDir.FirecrackerLogFifoPath() + //tc.expectedCfg.MetricsFifo = svc.shimDir.FirecrackerMetricsFifoPath() + tc.expectedCfg.LogPath = "/tmp/log__start.logs" drives := make([]models.Drive, tc.expectedStubDriveCount) for i := 0; i < tc.expectedStubDriveCount; i++ { From e0bf2b70bd33ce4e59926ce0d583ea9d916aeff5 Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Wed, 15 Jul 2020 05:42:46 -0400 Subject: [PATCH 5/9] Added dialer for firecracker socket Signed-off-by: Plamen Petrov --- runtime/service.go | 30 +++++++++++++++++++++++++++++- 1 file changed, 29 insertions(+), 1 deletion(-) diff --git a/runtime/service.go b/runtime/service.go index dde4a5e0c..7c0fd76bf 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -788,7 +788,10 @@ func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotReque s.logger.WithError(err).Error("startFirecrackerProcess returned an error") return nil, err } - time.Sleep(200 * time.Millisecond) + + if err := s.dialFirecrackerSocket(); err != nil { + s.logger.WithError(err).Error("Failed to wait for firecracker socket") + } s.createHTTPControlClient() loadSnapReq, err := formLoadSnapReq(req.SnapshotFilePath, req.MemFilePath) @@ -1711,3 +1714,28 @@ func (s *service) startFirecrackerProcess() error { return nil } + +func (s *service) dialFirecrackerSocket() error { + for { + var d net.Dialer + ctx, cancel := context.WithTimeout(context.Background(), 50*time.Millisecond) + defer cancel() + + c, err := d.DialContext(ctx, "unix", s.shimDir.FirecrackerSockPath()) + if err != nil { + if ctx.Err() != nil { + s.logger.WithError(ctx.Err()).Error("timed out while waiting for firecracker socket") + return ctx.Err() + } + + time.Sleep(1 * time.Millisecond) + continue + } + + c.Close() + + break + } + + return nil +} From c2c9057b88a79d2f3e0c70253c3c143029e783be Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Mon, 21 Sep 2020 10:42:24 +0300 Subject: [PATCH 6/9] kill shim functionality firecracker update Signed-off-by: Plamen Petrov --- Makefile | 4 +- _submodules/firecracker | 2 +- firecracker-control/local.go | 204 ++++++++++++++++++++++++++++++++++- runtime/service.go | 43 -------- 4 files changed, 205 insertions(+), 48 deletions(-) diff --git a/Makefile b/Makefile index 80f467cd8..812b61fda 100644 --- a/Makefile +++ b/Makefile @@ -278,8 +278,8 @@ demo-network: install-cni-bins $(FCNET_CONFIG) # Firecracker submodule ########################## .PHONY: firecracker -firecracker: $(FIRECRACKER_BIN) $(JAILER_BIN) - +firecracker: + _submodules/firecracker/tools/devtool build --release .PHONY: install-firecracker install-firecracker: firecracker install -D -o root -g root -m755 -t $(INSTALLROOT)/bin $(FIRECRACKER_BIN) diff --git a/_submodules/firecracker b/_submodules/firecracker index 1ee9e9365..9511525c8 160000 --- a/_submodules/firecracker +++ b/_submodules/firecracker @@ -1 +1 @@ -Subproject commit 1ee9e936537b950452eb75fc22a9c4301783b7f3 +Subproject commit 9511525c8d8abf836a45f922ca69ef18b04efa41 diff --git a/firecracker-control/local.go b/firecracker-control/local.go index 668e096fc..199eed466 100644 --- a/firecracker-control/local.go +++ b/firecracker-control/local.go @@ -71,6 +71,8 @@ type local struct { processesMu sync.Mutex processes map[string]int32 + + fcControlSocket *net.UnixListener } func newLocal(ic *plugin.InitContext) (*local, error) { @@ -243,7 +245,7 @@ func (s *local) StopVM(requestCtx context.Context, req *proto.StopVMRequest) (*e defer client.Close() resp, shimErr := client.StopVM(requestCtx, req) - waitErr := s.waitForShimToExit(requestCtx, req.VMID) + waitErr := s.waitForShimToExit(requestCtx, req.VMID, false) // Assuming the shim is returning containerd's error code, return the error as is if possible. if waitErr == nil { @@ -252,7 +254,7 @@ func (s *local) StopVM(requestCtx context.Context, req *proto.StopVMRequest) (*e return resp, multierror.Append(shimErr, waitErr).ErrorOrNil() } -func (s *local) waitForShimToExit(ctx context.Context, vmID string) error { +func (s *local) waitForShimToExit(ctx context.Context, vmID string, killShim bool) error { socketAddr, err := fcShim.SocketAddress(ctx, vmID) if err != nil { return err @@ -267,6 +269,17 @@ func (s *local) waitForShimToExit(ctx context.Context, vmID string) error { } defer delete(s.processes, socketAddr) + if killShim { + s.logger.Debug("Killing shim") + + if err := syscall.Kill(int(pid), 9); err != nil { + s.logger.WithError(err).Error("Failed to kill shim process") + return err + } + + return nil + } + return internal.WaitForPidToExit(ctx, stopVMInterval, pid) } @@ -464,6 +477,150 @@ func setShimOOMScore(shimPid int) error { return nil } +func (s *local) loadShim(ctx context.Context, ns, vmID, containerdAddress string) (*exec.Cmd, error) { + logger := s.logger.WithField("vmID", vmID) + logger.Debug("Loading shim") + + shimSocketAddress, err := fcShim.SocketAddress(ctx, vmID) + if err != nil { + err = errors.Wrap(err, "failed to obtain shim socket address") + s.logger.WithError(err).Error() + return nil, err + } + + shimSocket, err := shim.NewSocket(shimSocketAddress) + if isEADDRINUSE(err) { + return nil, status.Errorf(codes.AlreadyExists, "VM with ID %q already exists (socket: %q)", vmID, shimSocketAddress) + } else if err != nil { + err = errors.Wrapf(err, "failed to open shim socket at address %q", shimSocketAddress) + s.logger.WithError(err).Error() + return nil, err + } + + // If we're here, there is no pre-existing shim for this VMID, so we spawn a new one + defer shimSocket.Close() + if err := os.Mkdir(s.config.ShimBaseDir, 0700); err != nil && !os.IsExist(err) { + s.logger.WithError(err).Error() + return nil, errors.Wrapf(err, "failed to make shim base directory: %s", s.config.ShimBaseDir) + } + + shimDir, err := vm.ShimDir(s.config.ShimBaseDir, ns, vmID) + if err != nil { + err = errors.Wrapf(err, "failed to build shim path") + s.logger.WithError(err).Error() + return nil, err + } + + err = shimDir.Mkdir() + if err != nil { + err = errors.Wrapf(err, "failed to create VM dir %q", shimDir.RootPath()) + s.logger.WithError(err).Error() + return nil, err + } + + fcSocketAddress, err := fcShim.FCControlSocketAddress(ctx, vmID) + if err != nil { + err = errors.Wrap(err, "failed to obtain shim socket address") + s.logger.WithError(err).Error() + return nil, err + } + + fcSocket, err := shim.NewSocket(fcSocketAddress) + if err != nil { + err = errors.Wrapf(err, "failed to open fccontrol socket at address %q", fcSocketAddress) + s.logger.WithError(err).Error() + return nil, err + } + + s.fcControlSocket = fcSocket + + args := []string{ + "-namespace", ns, + "-address", containerdAddress, + } + + cmd := exec.Command(internal.ShimBinaryName, args...) + + // note: The working dir of the shim has an effect on the length of the path + // needed to specify various unix sockets that the shim uses to communicate + // with the firecracker VMM and guest agent within. The length of that path + // has a relatively low limit (usually 108 chars), so modifying the working + // dir should be done with caution. See internal/vm/dir.go for the path + // definitions. + cmd.Dir = shimDir.RootPath() + + shimSocketFile, err := shimSocket.File() + if err != nil { + err = errors.Wrap(err, "failed to get shim socket fd") + logger.WithError(err).Error() + return nil, err + } + + fcSocketFile, err := fcSocket.File() + if err != nil { + err = errors.Wrap(err, "failed to get shim fccontrol socket fd") + logger.WithError(err).Error() + return nil, err + } + + cmd.ExtraFiles = append(cmd.ExtraFiles, shimSocketFile, fcSocketFile) + fcSocketFDNum := 2 + len(cmd.ExtraFiles) // "2 +" because ExtraFiles come after stderr (fd #2) + + ttrpc := containerdAddress + ".ttrpc" + cmd.Env = append(os.Environ(), + fmt.Sprintf("%s=%s", ttrpcAddressEnv, ttrpc), + fmt.Sprintf("%s=%s", internal.VMIDEnvVarKey, vmID), + fmt.Sprintf("%s=%s", internal.FCSocketFDEnvKey, strconv.Itoa(fcSocketFDNum))) // TODO remove after containerd is updated to expose ttrpc server to shim + + cmd.SysProcAttr = &syscall.SysProcAttr{ + Setpgid: true, + } + + // shim stderr is just raw text, so pass it through our logrus formatter first + cmd.Stderr = logger.WithField("shim_stream", "stderr").WriterLevel(logrus.ErrorLevel) + // shim stdout on the other hand is already formatted by logrus, so pass that transparently through to containerd logs + cmd.Stdout = logger.Logger.Out + + logger.Debugf("starting %s", internal.ShimBinaryName) + + err = cmd.Start() + if err != nil { + err = errors.Wrap(err, "failed to start shim child process") + logger.WithError(err).Error() + return nil, err + } + + // make sure to wait after start + go func() { + if err := cmd.Wait(); err != nil { + if exitErr, ok := err.(*exec.ExitError); ok { + // shim is usually terminated by cancelling the context + logger.WithError(exitErr).Debug("shim has been terminated") + } else { + logger.WithError(err).Error("shim has been unexpectedly terminated") + } + } + + // Close all Unix abstract sockets. + if err := shimSocketFile.Close(); err != nil { + logger.WithError(err).Errorf("failed to close %q", shimSocketFile.Name()) + } + if err := fcSocketFile.Close(); err != nil { + logger.WithError(err).Errorf("failed to close %q", fcSocketFile.Name()) + } + }() + + err = setShimOOMScore(cmd.Process.Pid) + if err != nil { + logger.WithError(err).Error() + return nil, err + } + + s.addShim(shimSocketAddress, cmd) + + return cmd, nil +} + // PauseVM Pauses a VM func (s *local) PauseVM(ctx context.Context, req *proto.PauseVMRequest) (*empty.Empty, error) { client, err := s.shimFirecrackerClient(ctx, req.VMID) @@ -520,6 +677,18 @@ func (s *local) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotReq // LoadSnapshot Loads a snapshot of a VM func (s *local) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotRequest) (*empty.Empty, error) { + ns, err := namespaces.NamespaceRequired(ctx) + if err != nil { + err = errors.Wrap(err, "error retrieving namespace of request") + s.logger.WithError(err).Error() + return nil, err + } + + _, err = s.loadShim(ctx, ns, req.VMID, s.containerdAddress) + if err != nil { + return nil, err + } + client, err := s.shimFirecrackerClient(ctx, req.VMID) if err != nil { return nil, err @@ -553,5 +722,36 @@ func (s *local) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty. return nil, err } + s.fcControlSocket.Close() + + shimSocketAddress, err := fcShim.SocketAddress(ctx, req.VMID) + if err != nil { + err = errors.Wrap(err, "failed to obtain shim socket address") + s.logger.WithError(err).Error() + return nil, err + } + removeErr := os.RemoveAll(shimSocketAddress) + if removeErr != nil { + s.logger.Errorf("failed to remove shim socket addr file: %v", removeErr) + return nil, err + } + + fcSocketAddress, err := fcShim.FCControlSocketAddress(ctx, req.VMID) + if err != nil { + s.logger.Error("failed to get FC socket address") + return nil, err + } + removeErr = os.RemoveAll(fcSocketAddress) + if removeErr != nil { + s.logger.Errorf("failed to remove fc socket addr file: %v", removeErr) + return nil, err + } + + waitErr := s.waitForShimToExit(ctx, req.VMID, true) + if waitErr != nil { + s.logger.Error("failed to wait for shim to exit on offload") + return nil, waitErr + } + return resp, nil } diff --git a/runtime/service.go b/runtime/service.go index 7c0fd76bf..85e366dc8 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -811,24 +811,6 @@ func (s *service) LoadSnapshot(ctx context.Context, req *proto.LoadSnapshotReque return nil, err } - // Workaround for https://github.com/firecracker-microvm/firecracker/issues/1974 - patchDriveReq, err := formPatchDriveReq("MN2HE43UOVRDA", s.taskDrivePathOnHost) - if err != nil { - s.logger.WithError(err).Error("Failed to create patch drive request") - return nil, err - } - - resp, err = s.httpControlClient.Do(patchDriveReq) - if err != nil { - s.logger.WithError(err).Error("Failed to send patch drive request") - return nil, err - } - if !strings.Contains(resp.Status, "204") { - s.logger.WithError(err).Error("Failed to patch drive") - s.logger.WithError(err).Errorf("Status of request: %s", resp.Status) - return nil, err - } - return &empty.Empty{}, nil } @@ -1650,31 +1632,6 @@ func formCreateSnapReq(snapshotPath, memPath string) (*http.Request, error) { return req, nil } -func formPatchDriveReq(driveID, pathOnHost string) (*http.Request, error) { - var req *http.Request - - data := map[string]string{ - "drive_id": driveID, - "path_on_host": pathOnHost, - } - json, err := json.Marshal(data) - if err != nil { - logrus.WithError(err).Error("Failed to marshal json data") - return nil, err - } - - req, err = http.NewRequest("PATCH", fmt.Sprintf("http+unix://firecracker/drives/%s", driveID), bytes.NewBuffer(json)) - if err != nil { - logrus.WithError(err).Error("Failed to create new HTTP request in formPauseReq") - return nil, err - } - - req.Header.Add("accept", "application/json") - req.Header.Add("Content-Type", "application/json") - - return req, nil -} - func (s *service) startFirecrackerProcess() error { firecPath, err := exec.LookPath("firecracker") if err != nil { From 407e26a7c6434c1a204b8dd5e9f2ea028cc8c7ee Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Mon, 21 Sep 2020 17:07:54 +0300 Subject: [PATCH 7/9] Remove unnecessary mkdir * Check that shim dir exists when loading shim * No longer try to create shim dir when loading shim, as it must exist Signed-off-by: Plamen Petrov --- firecracker-control/local.go | 21 +++++---------------- 1 file changed, 5 insertions(+), 16 deletions(-) diff --git a/firecracker-control/local.go b/firecracker-control/local.go index 199eed466..973c5e0ed 100644 --- a/firecracker-control/local.go +++ b/firecracker-control/local.go @@ -272,7 +272,7 @@ func (s *local) waitForShimToExit(ctx context.Context, vmID string, killShim boo if killShim { s.logger.Debug("Killing shim") - if err := syscall.Kill(int(pid), 9); err != nil { + if err := syscall.Kill(int(pid), syscall.SIGKILL); err != nil { s.logger.WithError(err).Error("Failed to kill shim process") return err } @@ -442,10 +442,6 @@ func (s *local) newShim(ns, vmID, containerdAddress string, shimSocket *net.Unix if err := fcSocketFile.Close(); err != nil { logger.WithError(err).Errorf("failed to close %q", fcSocketFile.Name()) } - - if err := os.RemoveAll(shimDir.RootPath()); err != nil { - logger.WithError(err).Errorf("failed to remove %q", shimDir.RootPath()) - } }() err = setShimOOMScore(cmd.Process.Pid) @@ -497,11 +493,11 @@ func (s *local) loadShim(ctx context.Context, ns, vmID, containerdAddress string return nil, err } - // If we're here, there is no pre-existing shim for this VMID, so we spawn a new one defer shimSocket.Close() - if err := os.Mkdir(s.config.ShimBaseDir, 0700); err != nil && !os.IsExist(err) { - s.logger.WithError(err).Error() - return nil, errors.Wrapf(err, "failed to make shim base directory: %s", s.config.ShimBaseDir) + + // If we're here, there is no pre-existing shim for this VMID, so we spawn a new one + if _, err := os.Stat(s.config.ShimBaseDir); os.IsNotExist(err) { + return nil, errors.Wrapf(err, "shim base dir does not exist: %s", s.config.ShimBaseDir) } shimDir, err := vm.ShimDir(s.config.ShimBaseDir, ns, vmID) @@ -511,13 +507,6 @@ func (s *local) loadShim(ctx context.Context, ns, vmID, containerdAddress string return nil, err } - err = shimDir.Mkdir() - if err != nil { - err = errors.Wrapf(err, "failed to create VM dir %q", shimDir.RootPath()) - s.logger.WithError(err).Error() - return nil, err - } - fcSocketAddress, err := fcShim.FCControlSocketAddress(ctx, vmID) if err != nil { err = errors.Wrap(err, "failed to obtain shim socket address") From b78efd2c61b5f1e05a64f6b0c478958c15d52b3f Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Tue, 22 Sep 2020 12:42:38 +0300 Subject: [PATCH 8/9] Use SIGKILL, remove patchDrive artifacts Signed-off-by: Plamen Petrov --- runtime/service.go | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/runtime/service.go b/runtime/service.go index 85e366dc8..ab2d99d62 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -152,9 +152,8 @@ type service struct { vsockPortMu sync.Mutex // httpControlClient is to send pause/resume/snapshot commands to the microVM - httpControlClient *http.Client - firecrackerPid int - taskDrivePathOnHost string + httpControlClient *http.Client + firecrackerPid int } func shimOpts(shimCtx context.Context) (*shim.Opts, error) { @@ -838,7 +837,7 @@ func (s *service) CreateSnapshot(ctx context.Context, req *proto.CreateSnapshotR // Offload Shuts down a VM and deletes the corresponding firecracker socket // and vsock. All of the other resources will persist func (s *service) Offload(ctx context.Context, req *proto.OffloadRequest) (*empty.Empty, error) { - if err := syscall.Kill(s.firecrackerPid, 9); err != nil { + if err := syscall.Kill(s.firecrackerPid, syscall.SIGKILL); err != nil { s.logger.WithError(err).Error("Failed to kill firecracker process") return nil, err } @@ -1061,8 +1060,6 @@ func (s *service) Create(requestCtx context.Context, request *taskAPI.CreateTask } rootfsMnt := request.Rootfs[0] - s.taskDrivePathOnHost = rootfsMnt.Source - err = s.containerStubHandler.Reserve(requestCtx, request.ID, rootfsMnt.Source, vmBundleDir.RootfsPath(), "ext4", nil, s.driveMountClient, s.machine) if err != nil { From e2520f309dfa58633f38bfc9b4099b80201bb03d Mon Sep 17 00:00:00 2001 From: Plamen Petrov Date: Tue, 22 Sep 2020 14:46:39 +0300 Subject: [PATCH 9/9] store firecracker logs in shimDir Signed-off-by: Plamen Petrov --- internal/common.go | 6 ++++++ internal/vm/dir.go | 13 +++++++++++++ runtime/service.go | 14 ++++---------- runtime/service_test.go | 2 +- 4 files changed, 24 insertions(+), 11 deletions(-) diff --git a/internal/common.go b/internal/common.go index 9e7a78b48..cb8cfb7a9 100644 --- a/internal/common.go +++ b/internal/common.go @@ -50,6 +50,12 @@ const ( // ShimLogFifoName is the name of the FIFO created by containerd for a shim to write its logs to ShimLogFifoName = "log" + // LogPathNameStart is the name of the FIFO created by containerd for a shim to write its logs to + LogPathNameStart = "log_start" + + // LogPathNameLoad is the name of the FIFO created by containerd for a shim to write its logs to + LogPathNameLoad = "log_load" + // OCIConfigName is the name of the OCI bundle's config field OCIConfigName = "config.json" diff --git a/internal/vm/dir.go b/internal/vm/dir.go index 228350a35..6c3fdece7 100644 --- a/internal/vm/dir.go +++ b/internal/vm/dir.go @@ -83,6 +83,19 @@ func (d Dir) OpenLogFifo(requestCtx context.Context) (io.ReadWriteCloser, error) return fifo.OpenFifo(requestCtx, d.LogFifoPath(), unix.O_WRONLY|unix.O_NONBLOCK, 0200) } +// LogStartPath returns the path to the file for storing +// firecracker logs after the microVM is started and until +// it is Offloaded +func (d Dir) LogStartPath() string { + return filepath.Join(d.RootPath(), internal.LogPathNameStart) +} + +// LogLoadPath returns the path to the file for storing +// firecracker logs after the microVM is loaded from a snapshot +func (d Dir) LogLoadPath() string { + return filepath.Join(d.RootPath(), internal.LogPathNameLoad) +} + // FirecrackerSockPath returns the path to the unix socket at which the firecracker VMM // services its API func (d Dir) FirecrackerSockPath() string { diff --git a/runtime/service.go b/runtime/service.go index ab2d99d62..b7c456696 100644 --- a/runtime/service.go +++ b/runtime/service.go @@ -875,13 +875,8 @@ func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker } // TODO: Remove hardcoding and make a parameter - logFilePath := fmt.Sprintf("/tmp/log_%s_start.logs", s.vmID) - if err := os.RemoveAll(logFilePath); err != nil { - s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) - return nil, err - } - if _, err := os.OpenFile(logFilePath, os.O_RDONLY|os.O_CREATE, 0600); err != nil { - s.logger.WithError(err).Errorf("Failed to create %s", logFilePath) + if _, err := os.OpenFile(s.shimDir.LogStartPath(), os.O_RDONLY|os.O_CREATE, 0600); err != nil { + s.logger.WithError(err).Errorf("Failed to create %s", s.shimDir.LogStartPath()) return nil, err } @@ -892,7 +887,7 @@ func (s *service) buildVMConfiguration(req *proto.CreateVMRequest) (*firecracker ID: "agent_api", }}, // Put LogPath insteadof LogFifo here to comply with the new Firecracker logging - LogPath: logFilePath, + LogPath: s.shimDir.LogStartPath(), MachineCfg: machineConfigurationFromProto(s.config, req.MachineCfg), LogLevel: s.config.DebugHelper.GetFirecrackerLogLevel(), VMID: s.vmID, @@ -1636,8 +1631,7 @@ func (s *service) startFirecrackerProcess() error { return err } - // TODO: Remove hardcoding and make a parameter - logFilePath := fmt.Sprintf("/tmp/log_%s_after.logs", s.vmID) + logFilePath := s.shimDir.LogLoadPath() if err := os.RemoveAll(logFilePath); err != nil { s.logger.WithError(err).Errorf("Failed to delete %s", logFilePath) return err diff --git a/runtime/service_test.go b/runtime/service_test.go index 80e690221..b10417dd3 100644 --- a/runtime/service_test.go +++ b/runtime/service_test.go @@ -260,7 +260,7 @@ func TestBuildVMConfiguration(t *testing.T) { // TODO: FIX TEST WHEN LogPath is no longer hardcoded //tc.expectedCfg.LogFifo = svc.shimDir.FirecrackerLogFifoPath() //tc.expectedCfg.MetricsFifo = svc.shimDir.FirecrackerMetricsFifoPath() - tc.expectedCfg.LogPath = "/tmp/log__start.logs" + tc.expectedCfg.LogPath = svc.shimDir.LogStartPath() drives := make([]models.Drive, tc.expectedStubDriveCount) for i := 0; i < tc.expectedStubDriveCount; i++ {