diff --git a/Justfile b/Justfile index cfd7891..7bca018 100644 --- a/Justfile +++ b/Justfile @@ -12,7 +12,7 @@ run: sudo -E capsh --keep=1 --user=$USER --inh=cap_net_admin --addamb=cap_net_admin -- -c \ 'RUST_BACKTRACE=1 '$CARGO_PATH' run --bin vmm -- cli --memory 512 --cpus 1 \ --kernel tools/kernel/linux-cloud-hypervisor/arch/x86/boot/compressed/vmlinux.bin \ - --iface-host-addr 172.29.0.1 --netmask 255.255.0.0 --iface-guest-addr 172.29.0.2 \ + --iface-host-addr 172.29.0.1 --netmask 255.255.0.0 --network 172.29.0.0 --iface-guest-addr 172.29.0.2 \ --initramfs=tools/rootfs/initramfs.img' build-kernel: diff --git a/src/agent/examples/config.toml b/src/agent/examples/config.toml index 013193c..691b100 100644 --- a/src/agent/examples/config.toml +++ b/src/agent/examples/config.toml @@ -7,5 +7,5 @@ address = "localhost" port = 50051 [build] -source-code-path = "CHANGEME/src/agent/examples/main.rs" +source-code-path = "/home/spierrot/git/do/cloudlet/src/agent/examples/main.rs" release = true diff --git a/src/vmm/Cargo.toml b/src/vmm/Cargo.toml index bdda86e..c57316c 100644 --- a/src/vmm/Cargo.toml +++ b/src/vmm/Cargo.toml @@ -15,6 +15,7 @@ clap-verbosity-flag = "2.2.0" epoll = "4.3.3" event-manager = { version = "0.4.0", features = ["remote_endpoint"] } futures = "0.3.30" +iptables = "0.5.1" kvm-bindings = { version = "0.7.0", features = ["fam-wrappers"] } kvm-ioctls = "0.16.0" libc = "0.2.153" diff --git a/src/vmm/src/args.rs b/src/vmm/src/args.rs index 3feceef..3759420 100644 --- a/src/vmm/src/args.rs +++ b/src/vmm/src/args.rs @@ -44,6 +44,10 @@ pub struct CliArguments { #[clap(long, env, required = true)] pub iface_host_addr: Ipv4Addr, + /// Network. + #[clap(long, env, required = true)] + pub network: Ipv4Addr, + /// Subnet mask for network. #[clap(long, env, required = true)] pub netmask: Ipv4Addr, diff --git a/src/vmm/src/core/devices/virtio/net/bridge.rs b/src/vmm/src/core/devices/virtio/net/bridge.rs index 30c0c78..6412cb4 100644 --- a/src/vmm/src/core/devices/virtio/net/bridge.rs +++ b/src/vmm/src/core/devices/virtio/net/bridge.rs @@ -1,59 +1,137 @@ +use std::net::{IpAddr, Ipv4Addr}; + use futures::stream::TryStreamExt; use rtnetlink::{new_connection, Error, Handle}; -pub fn host_bridge(tap_name: String, bridge_name: String) { - let (connection, handle, _) = new_connection().unwrap(); - tokio::spawn(connection); +use super::xx_netmask_width; - futures::executor::block_on(async { - let _ = create_bridge_if_not_exist(handle.clone(), bridge_name.clone()).await; - let _ = attach_link_to_bridge(handle, tap_name, bridge_name).await; - }) +#[derive(Clone)] +pub struct Bridge { + name: String, + handle: Handle, } -async fn create_bridge_if_not_exist(handle: Handle, name: String) -> Result<(), Error> { - let mut bridge_names = handle.link().get().match_name(name.clone()).execute(); - - match bridge_names.try_next().await? { - Some(_) => Ok(()), - None => handle - .link() - .add() - .bridge(name) - .execute() - .await - .map_err(|_| Error::RequestFailed), +impl Bridge { + pub fn new(name: String) -> Self { + let (connection, handle, _) = new_connection().unwrap(); + tokio::spawn(connection); + + let br = Self { name, handle }; + let _ = br.create_bridge_if_not_exist(); + + br } -} -async fn attach_link_to_bridge( - handle: Handle, - link_name: String, - master_name: String, -) -> Result<(), Error> { - let mut link_names = handle.link().get().match_name(link_name.clone()).execute(); - let mut master_names = handle - .link() - .get() - .match_name(master_name.clone()) - .execute(); - - let link_index = match link_names.try_next().await? { - Some(link) => link.header.index, - None => panic!(), - }; - let master_index = match master_names.try_next().await? { - Some(link) => link.header.index, - None => panic!(), - }; - - let _ = handle - .link() - .set(link_index) - .controller(master_index) - .execute() - .await - .map_err(|_| Error::RequestFailed); - - Ok(()) + fn create_bridge_if_not_exist(&self) { + futures::executor::block_on(async { + let mut bridge_names = self + .handle + .link() + .get() + .match_name(self.name.clone()) + .execute(); + + let _ = match bridge_names.try_next().await { + Ok(_) => Ok(()), + Err(_) => self + .handle + .link() + .add() + .bridge(self.name.clone()) + .execute() + .await + .map_err(|_| Error::RequestFailed), + }; + }); + } + + pub fn set_addr(&self, addr: Ipv4Addr, netmask: Ipv4Addr) { + futures::executor::block_on(async { + let mut bridge_names = self + .handle + .link() + .get() + .match_name(self.name.clone()) + .execute(); + + let bridge_index = match bridge_names.try_next().await { + Ok(Some(link)) => link.header.index, + Ok(None) => panic!(), + Err(_) => panic!(), + }; + + let prefix_len = xx_netmask_width(netmask.octets()); + + let _ = self + .handle + .address() + .add(bridge_index, IpAddr::V4(addr), prefix_len) + .execute() + .await + .map_err(|_| Error::RequestFailed); + }); + } + + pub fn set_up(&self) { + futures::executor::block_on(async { + let mut bridge_names = self + .handle + .link() + .get() + .match_name(self.name.clone()) + .execute(); + + let bridge_index = match bridge_names.try_next().await { + Ok(Some(link)) => link.header.index, + Ok(None) => panic!(), + Err(_) => panic!(), + }; + + let _ = self + .handle + .link() + .set(bridge_index) + .up() + .execute() + .await + .map_err(|_| Error::RequestFailed); + }); + } + + pub fn attach_link(&self, link_name: String) { + futures::executor::block_on(async { + let mut link_names = self + .handle + .link() + .get() + .match_name(link_name.clone()) + .execute(); + let mut master_names = self + .handle + .link() + .get() + .match_name(self.name.clone()) + .execute(); + + let link_index = match link_names.try_next().await { + Ok(Some(link)) => link.header.index, + Ok(None) => panic!(), + Err(_) => panic!(), + }; + let master_index = match master_names.try_next().await { + Ok(Some(link)) => link.header.index, + Ok(None) => panic!(), + Err(_) => panic!(), + }; + + let _ = self + .handle + .link() + .set(link_index) + .controller(master_index) + .execute() + .await + .map_err(|_| Error::RequestFailed); + }); + } } diff --git a/src/vmm/src/core/devices/virtio/net/device.rs b/src/vmm/src/core/devices/virtio/net/device.rs index 94b5199..3698d84 100644 --- a/src/vmm/src/core/devices/virtio/net/device.rs +++ b/src/vmm/src/core/devices/virtio/net/device.rs @@ -1,4 +1,5 @@ -use super::bridge::host_bridge; +use super::bridge::Bridge; +use super::iptables::iptables_ip_masq; use super::queue_handler::QueueHandler; use super::{ simple_handler::SimpleHandler, tuntap::tap::Tap, Error, Result, NET_DEVICE_ID, @@ -35,6 +36,7 @@ pub struct Net { mem: Arc, pub config: Config, tap: Arc>, + _bridge: Bridge, } impl Net { @@ -43,7 +45,8 @@ impl Net { mem: Arc, device_mgr: Arc>, mmio_cfg: MmioConfig, - tap_addr: Ipv4Addr, + iface_host_addr: Ipv4Addr, + network: Ipv4Addr, netmask: Ipv4Addr, iface_guest_addr: Ipv4Addr, irq: u32, @@ -75,32 +78,38 @@ impl Net { // Set offload flags to match the relevant virtio features of the device (for now, // statically set in the constructor. - let tap = open_tap(None, Some(tap_addr), Some(netmask), &mut None, None, None) - .map_err(Error::TunTap)?; + let tap = open_tap(None, None, None, &mut None, None, None).map_err(Error::TunTap)?; // The layout of the header is specified in the standard and is 12 bytes in size. We // should define this somewhere. tap.set_vnet_hdr_size(VIRTIO_NET_HDR_SIZE as i32) .map_err(Error::Tap)?; + let bridge = Bridge::new("br0".to_string()); + bridge.set_addr(iface_host_addr, netmask); + bridge.attach_link(tap.get_name().map_err(Error::Tap)?); + bridge.set_up(); + + // Get internet access + iptables_ip_masq(network, netmask); + let net = Arc::new(Mutex::new(Net { mem, config: cfg, tap: Arc::new(Mutex::new(tap.clone())), + _bridge: bridge, })); let vmmio_param = register_mmio_device(mmio_cfg, device_mgr, irq, None, net.clone()) .map_err(Error::Virtio)?; let ip_pnp_param: String = format!( "ip={}::{}:{}::eth0:off", - iface_guest_addr, tap_addr, netmask + iface_guest_addr, iface_host_addr, netmask ); cmdline_extra_parameters.push(vmmio_param); cmdline_extra_parameters.push(ip_pnp_param); - host_bridge(tap.get_name().map_err(Error::Tap)?, "br0".to_string()); - Ok(net) } } diff --git a/src/vmm/src/core/devices/virtio/net/iptables.rs b/src/vmm/src/core/devices/virtio/net/iptables.rs new file mode 100644 index 0000000..bbe22f5 --- /dev/null +++ b/src/vmm/src/core/devices/virtio/net/iptables.rs @@ -0,0 +1,16 @@ +use std::net::Ipv4Addr; + +use super::xx_netmask_width; + +pub fn iptables_ip_masq(network: Ipv4Addr, netmask: Ipv4Addr) { + let prefix_len = xx_netmask_width(netmask.octets()); + let source = format!("{}/{}", network.to_string(), prefix_len); + + let ipt = iptables::new(false).unwrap(); + let rule = format!("-s {} ! -o br0 -j MASQUERADE", source); + + let exists = ipt.exists("nat", "POSTROUTING", rule.as_str()).unwrap(); + if !exists { + let _ = ipt.insert_unique("nat", "POSTROUTING", rule.as_str(), 1); + } +} diff --git a/src/vmm/src/core/devices/virtio/net/mod.rs b/src/vmm/src/core/devices/virtio/net/mod.rs index c254ae1..1277932 100644 --- a/src/vmm/src/core/devices/virtio/net/mod.rs +++ b/src/vmm/src/core/devices/virtio/net/mod.rs @@ -1,5 +1,6 @@ mod bridge; pub mod device; +pub mod iptables; mod queue_handler; mod simple_handler; pub mod tuntap; @@ -21,3 +22,7 @@ pub enum Error { } pub type Result = std::result::Result; + +pub fn xx_netmask_width(netmask: [u8; SZ]) -> u8 { + netmask.iter().map(|x| x.count_ones() as u8).sum() +} diff --git a/src/vmm/src/core/vmm.rs b/src/vmm/src/core/vmm.rs index c111a4c..c1a2121 100644 --- a/src/vmm/src/core/vmm.rs +++ b/src/vmm/src/core/vmm.rs @@ -58,7 +58,8 @@ pub struct VMM { event_mgr: EventMgr, vcpus: Vec, - tap_addr: Ipv4Addr, + iface_host_addr: Ipv4Addr, + network: Ipv4Addr, netmask: Ipv4Addr, iface_guest_addr: Ipv4Addr, net_devices: Vec>>, @@ -69,7 +70,12 @@ pub struct VMM { impl VMM { /// Create a new VMM. - pub fn new(tap_addr: Ipv4Addr, netmask: Ipv4Addr, iface_guest_addr: Ipv4Addr) -> Result { + pub fn new( + iface_host_addr: Ipv4Addr, + network: Ipv4Addr, + netmask: Ipv4Addr, + iface_guest_addr: Ipv4Addr, + ) -> Result { // Open /dev/kvm and get a file descriptor to it. let kvm = Kvm::new().map_err(Error::KvmIoctl)?; @@ -111,7 +117,8 @@ impl VMM { )), slip_pty: Arc::new(Mutex::new(slip_pty)), epoll, - tap_addr, + iface_host_addr, + network, netmask, iface_guest_addr, net_devices: Vec::new(), @@ -382,7 +389,8 @@ impl VMM { mem, self.device_mgr.clone(), mmio_cfg, - self.tap_addr, + self.iface_host_addr, + self.network, self.netmask, self.iface_guest_addr, irq, diff --git a/src/vmm/src/grpc/server.rs b/src/vmm/src/grpc/server.rs index e5c6919..6383c29 100644 --- a/src/vmm/src/grpc/server.rs +++ b/src/vmm/src/grpc/server.rs @@ -51,6 +51,7 @@ impl VmmServiceTrait for VmmService { const HOST_IP: Ipv4Addr = Ipv4Addr::new(172, 29, 0, 1); const HOST_NETMASK: Ipv4Addr = Ipv4Addr::new(255, 255, 0, 0); + const NETWORK: Ipv4Addr = Ipv4Addr::new(172, 29, 0, 0); const GUEST_IP: Ipv4Addr = Ipv4Addr::new(172, 29, 0, 2); // get current directory @@ -157,7 +158,8 @@ impl VmmServiceTrait for VmmService { } let initramfs_path = PathBuf::from(&initramfs_entire_file_path); - let mut vmm = VMM::new(HOST_IP, HOST_NETMASK, GUEST_IP).map_err(VmmErrors::VmmNew)?; + let mut vmm = + VMM::new(HOST_IP, NETWORK, HOST_NETMASK, GUEST_IP).map_err(VmmErrors::VmmNew)?; // Configure the VMM parameters might need to be calculated rather than hardcoded vmm.configure(1, 4000, kernel_path, &Some(initramfs_path)) diff --git a/src/vmm/src/main.rs b/src/vmm/src/main.rs index 7f6a748..364a377 100644 --- a/src/vmm/src/main.rs +++ b/src/vmm/src/main.rs @@ -42,6 +42,7 @@ async fn main() -> Result<(), Box> { // Create a new VMM let mut vmm = VMM::new( cli_args.iface_host_addr, + cli_args.network, cli_args.netmask, cli_args.iface_guest_addr, )