Skip to content

Commit

Permalink
Merge pull request #64 from holochain/turn-server
Browse files Browse the repository at this point in the history
feat(nixosConfigurations): add new machine serving {turn,signal,bootstrap}.infra.holochain.org
  • Loading branch information
steveej authored Mar 21, 2024
2 parents 3082865 + fdfd132 commit c73d464
Show file tree
Hide file tree
Showing 14 changed files with 1,210 additions and 80 deletions.
559 changes: 521 additions & 38 deletions flake.lock

Large diffs are not rendered by default.

37 changes: 29 additions & 8 deletions flake.nix
Original file line number Diff line number Diff line change
Expand Up @@ -2,7 +2,8 @@
description = "The new, performant, and simplified version of Holochain on Rust (sometimes called Holochain RSM for Refactored State Model) ";

inputs = {
nixpkgs = {url = "github:nixos/nixpkgs/release-23.05";};
nixpkgs.follows = "nixpkgs-23-11";
nixpkgs-23-11 = {url = "github:nixos/nixpkgs/nixos-23.11";};
nixpkgsGithubActionRunners = {url = "github:nixos/nixpkgs/nixos-unstable";};
nixpkgsUnstable = {url = "github:nixos/nixpkgs/nixos-unstable";};
nixpkgsMaster = {url = "github:nixos/nixpkgs/master";};
Expand Down Expand Up @@ -32,6 +33,11 @@
sops-nix.url = "github:Mic92/sops-nix";
sops-nix.inputs.nixpkgs.follows = "nixpkgs";

crane = {
url = "github:ipetkov/crane";
inputs.nixpkgs.follows = "nixpkgs-23-11";
};

keys_steveej = {
url = "https://github.com/steveej.keys";
flake = false;
Expand Down Expand Up @@ -63,17 +69,30 @@
flake = false;
};

# NAR mismatch as of 2023/07/21
# keys_zippy = {
# url = "https://github.com/zippy.keys";
# flake = false;
# };
keys_zippy = {
url = "https://github.com/zippy.keys";
flake = false;
};
keys_artbrock = {
url = "https://github.com/artbrock.keys";
flake = false;
};

cachix_for_watch_store.url = github:cachix/cachix/v1.5;
cachix_for_watch_store.url = "github:cachix/cachix/v1.5";

tx5.url = "github:holochain/tx5/tx5-signal-srv-v0.0.7-alpha";
tx5.flake = false;

holochain-versions.url = "github:holochain/holochain?dir=versions/weekly";
holochain = {
url = "github:holochain/holochain";
inputs.versions.follows = "holochain-versions";
};

coturn = {
flake = false;
url = "github:steveej-forks/coturn/debug-cli-login";
};
};

outputs = inputs @ {
Expand Down Expand Up @@ -155,7 +174,9 @@
};

packages = {
nomad = inputs'.nixpkgsMaster.legacyPackages.nomad_1_6;
nomad = inputs'.nixpkgs.legacyPackages.nomad_1_6;

nixos-anywhere = inputs'.nixos-anywhere.packages.default;
};
};
flake = {
Expand Down
1 change: 1 addition & 0 deletions modules/flake-parts/apps.ssh-/default.nix
Original file line number Diff line number Diff line change
Expand Up @@ -6,6 +6,7 @@
}: {
perSystem = {pkgs, ...}: let
mkSsh = {
attrName,
hostName,
deployUser,
}:
Expand Down
186 changes: 186 additions & 0 deletions modules/flake-parts/holochain-turn-server.nix
Original file line number Diff line number Diff line change
@@ -0,0 +1,186 @@
{
# System independent arguments.
self,
lib,
...
}: {
flake = {
# system independent outputs like nixosModules, nixosConfigurations, etc.

# nixosConfigurations.example-host = ...
overlays.coturn = _final: previous: {
coturn = previous.coturn.overrideAttrs (
_super: {
src = self.inputs.coturn;
# coturn for NixOS needs to be built without libev_ok, otherwise acme-redirect won't work
LIBEV_OK = "0";
meta.platforms = lib.platforms.linux;
}
);
};

nixosModules.holochain-turn-server = {
config,
lib,
...
}: let
cfg = config.services.holochain-turn-server;
in {
options.services.holochain-turn-server = {
enable = lib.mkEnableOption "holochain turn server";
url = lib.mkOption {
description = "publicly visible url for the turn server";
type = lib.types.str;
};
turn-cert-dir = lib.mkOption {
description = "directory where fullchain.pem and key.pem are expected to exist";
type = lib.types.str;
default = config.security.acme.certs.${cfg.url}.directory;
};
address = lib.mkOption {
description = "address coturn should listen on";
type = lib.types.str;
};

nginx-http-port = lib.mkOption {
description = "port for nginx to listen on for answering ACME challenges";
type = lib.types.int;
# skipping 81 because it's the default coturn alternative http port
default = 82;
};

coturn-min-port = lib.mkOption {
description = "lower port for coturn's range";
type = lib.types.int;
default = 20000;
};

coturn-max-port = lib.mkOption {
description = "upper port for coturn's range";
type = lib.types.int;
default = 65535; # which is default but here listing explicitly
};

verbose = lib.mkEnableOption "verbose logging";

acme-redirect = lib.mkOption {
description = "value passed to acme-redirect configuration option";
type = lib.types.str;
default = "http://acme-${cfg.url}/.well-known/acme-challenge/";
};

username = lib.mkOption {
description = "user for establishing turn connections to coturn";
type = lib.types.str;
default = "test";
};

credential = lib.mkOption {
description = "credential for establishing turn connections to coturn";
type = lib.types.str;
default = "test";
};

extraCoturnAttrs = lib.mkOption {
description = "extra attributes assigned to services.coturn";
type = lib.types.attrs;
default = {};
};
};

config = lib.mkIf cfg.enable {
nixpkgs.overlays = [self.overlays.coturn];

networking.firewall.allowedTCPPorts = [
80
443
9641 # prometheus

cfg.nginx-http-port
];
networking.firewall.allowedUDPPorts = [
80
443
9641 # prometheus
];
networking.firewall.allowedUDPPortRanges = [
{
from = cfg.coturn-min-port;
to = cfg.coturn-max-port;
}
];

services.coturn =
{
enable = true;
listening-port = 80;
tls-listening-port = 443;
listening-ips = [cfg.address];
lt-cred-mech = true; # Use long-term credential mechanism.
realm = cfg.url;
cert = "${cfg.turn-cert-dir}/fullchain.pem";
pkey = "${cfg.turn-cert-dir}/key.pem";
no-cli = false;
min-port = cfg.coturn-min-port;
max-port = cfg.coturn-max-port;
extraConfig =
''
no-software-attribute
no-multicast-peers
no-tlsv1
no-tlsv1_1
user=${cfg.username}:${cfg.credential}
prometheus
''
+ lib.strings.optionalString cfg.verbose ''
verbose
''
+ lib.strings.optionalString (cfg.acme-redirect != null) ''
acme-redirect=${cfg.acme-redirect}
'';
}
// cfg.extraCoturnAttrs;

systemd.services.coturn.serviceConfig = {
LimitNOFILESoft = 10000;
};

# Add turnserver user to nginx group, because turnserver needs to have access to TLS certs from /var/lib/acme/
users.groups.nginx.members = ["turnserver"];

services.nginx = {
enable = true;

# the sole purpose of nginx here is TLS certificate renewal from letsencrypt
# coturn redirects ACME, i.e. HTTP GET requests matching '^/.well-known/acme-challenge/(.*)'
# to acme-turn.holo.host, which is intercepted by a reverse-proxy and redirected to port ${cfg.nginx-http-port} on this host
virtualHosts."${cfg.url}" = {
listen = [
{
addr = "${cfg.address}";
port = cfg.nginx-http-port;
ssl = false;
}
];
enableACME = true;
serverName = cfg.url;
};
};

security.acme = {
acceptTerms = true;
defaults = {
email = "acme@holo.host";
};

# after certificate renewal by acme coturn.service needs to reload this new cert, too
# see https://github.com/NixOS/nixpkgs/blob/nixos-23.05/nixos/modules/security/acme/default.nix#L322
certs."${cfg.url}".reloadServices = ["coturn"];

# staging server has higher retry limits. uncomment the following when debugging ACME challenges.
# certs."${cfg.url}".server = "https://acme-staging-v02.api.letsencrypt.org/directory";
};
};
};
};
}
Original file line number Diff line number Diff line change
Expand Up @@ -7,7 +7,6 @@
...
}: let
ipv4 = "5.78.43.185";
ipv6Prefix = "2a01:4ff:1f0:872a";
fqdn2domain = "infra.holochain.org";
in {
imports = [
Expand Down Expand Up @@ -37,36 +36,31 @@ in {
"holochain-ci.cachix.org-1:5IUSkZc0aoRS53rfkvH9Kid40NpyjwCMCzwRTXy+QN8="
];

boot.loader.grub = {
efiSupport = false;
device = "/dev/sda";
};
# boot.loader.systemd-boot.enable = true;
# boot.loader.efi.canTouchEfiVariables = true;
boot.kernelPackages = pkgs.linuxPackages_latest;

systemd.network.networks."10-uplink".networkConfig.Address = "${ipv6Prefix}::1/64";
boot.loader.systemd-boot.enable = false;
boot.loader.grub.efiSupport = true;
boot.loader.grub.efiInstallAsRemovable = false;

disko.devices.disk.sda = {
device = "/dev/sda";
type = "disk";
content = {
type = "table";
format = "gpt";
partitions = [
{
name = "boot";
start = "0";
end = "1M";
part-type = "primary";
flags = ["bios_grub"];
}
{
name = "root";
start = "1M";
end = "100%";
part-type = "primary";
bootable = true;
type = "gpt";
partitions = {
boot = {
size = "1M";
type = "EF02"; # for grub MBR
};
ESP = {
type = "EF00";
size = "1G";
content = {
type = "filesystem";
format = "vfat";
mountpoint = "/boot";
};
};
root = {
size = "100%";
content = {
type = "btrfs";
extraArgs = ["-f"]; # Override existing partition
Expand All @@ -77,15 +71,16 @@ in {
};
"/nix" = {
mountOptions = ["noatime"];
mountpoint = "/nix";
};
};
};
}
];
};
};
};
};

system.stateVersion = "23.05";
system.stateVersion = "23.11";

### ZeroTier
services.zerotierone = {
Expand All @@ -94,6 +89,7 @@ in {
nixpkgs.config.allowUnfreePredicate = pkg:
builtins.elem (lib.getName pkg) [
"zerotierone"
"nomad"
];

sops.secrets.zerotieroneNetworks = {
Expand Down Expand Up @@ -152,6 +148,7 @@ in {

### BIND and ACME

# FIXME: changes to the bind zone require a manual `systemctl restart bind`
system.activationScripts.bind-zones.text = ''
mkdir -p /etc/bind/zones
chown named:named /etc/bind/zones
Expand Down Expand Up @@ -186,6 +183,10 @@ in {
amsterdam2023.events.${fqdn2domain}. A 10.1.3.187
sj-bm-hostkey0.dev.${fqdn2domain}. A 185.130.224.33
turn.${fqdn2domain}. A ${self.nixosConfigurations.turn-infra-holochain-org.config.services.holochain-turn-server.address}
signal.${fqdn2domain}. A ${self.nixosConfigurations.turn-infra-holochain-org.config.services.tx5-signal-server.address}
bootstrap.${fqdn2domain}. A ${self.nixosConfigurations.turn-infra-holochain-org.config.services.kitsune-bootstrap.address}
'';
};

Expand Down Expand Up @@ -306,6 +307,12 @@ in {
# reverse_proxy https://holochain-ci.cachix.org
'';
};

"acme-turn.${fqdn2domain}:80" = {
extraConfig = ''
reverse_proxy http://turn.${fqdn2domain}:${builtins.toString self.nixosConfigurations.turn-infra-holochain-org.config.services.holochain-turn-server.nginx-http-port}
'';
};
};

sops.secrets.global-server-nomad-key = {
Expand All @@ -316,7 +323,7 @@ in {

services.nomad = {
enable = true;
package = self.packages.${pkgs.system}.nomad;
package = pkgs.nomad_1_6;
enableDocker = false;
dropPrivileges = false;

Expand Down
Loading

0 comments on commit c73d464

Please sign in to comment.