add proxy support for profiles!

pass proxy as ssh://<auth> instead of separate --sshProxyLogin flag

backend/btrixcloud/crawlmanager.py

@@ -34,6 +34,7 @@ async def run_profile_browser(
         crawler_image: str,
         baseprofile: str = "",
         profile_filename: str = "",
+        proxy_id: str = "",
     ) -> str:
         """run browser for profile creation"""
 
@@ -55,6 +56,7 @@ async def run_profile_browser(
             "vnc_password": secrets.token_hex(16),
             "expire_time": to_k8s_date(dt_now() + timedelta(seconds=30)),
             "crawler_image": crawler_image,
+            "proxy_id": proxy_id,
         }
 
         data = self.templates.env.get_template("profile_job.yaml").render(params)

backend/btrixcloud/operator/profiles.py

@@ -49,6 +49,12 @@ async def sync_profile_browsers(self, data: MCSyncData):
         params["profile_filename"] = spec.get("profileFilename", "")
         params["crawler_image"] = spec["crawlerImage"]
 
+        proxy_id = spec.get("proxyId")
+        params["proxy_id"] = proxy_id
+        if proxy_id:
+            proxy = self.crawl_config_ops.get_crawler_proxy(proxy_id)
+            params["ssh_proxy_auth"] = proxy.auth if proxy else ""
+
         params["url"] = spec.get("startUrl", "about:blank")
         params["vnc_password"] = spec.get("vncPassword")
 

backend/btrixcloud/profiles.py

@@ -115,6 +115,7 @@ async def create_new_browser(
             crawler_image=crawler_image,
             baseprofile=prev_profile_id,
             profile_filename=prev_profile_path,
+            proxy_id=profile_launch.proxyId or "",
         )
 
         if not browserid:

chart/app-templates/crawler.yaml

@@ -136,8 +136,8 @@ spec:
         - "@{{ profile_filename }}"
       {% endif %}
       {% if proxy_id %}
-        - --sshProxyLogin
-        - "{{ ssh_proxy_auth }}"
+        - --proxyServer
+        - "ssh://{{ ssh_proxy_auth }}"
         - --sshProxyPrivateKeyFile
         - /tmp/ssh-proxy/private-key
         - --sshProxyKnownHostsFile

chart/app-templates/profile_job.yaml

@@ -28,6 +28,8 @@ spec:
   profileFilename: "{{ profile_filename }}"
   vncPassword: "{{ vnc_password }}"
 
+  proxyId: "{{ proxy_id }}"
+
   {% if expire_time %}
   expireTime: "{{ expire_time }}"
   {% endif %}

chart/app-templates/profilebrowser.yaml

@@ -26,6 +26,13 @@ spec:
       emptyDir:
         sizeLimit: {{ profile_browser_workdir_size }}
 
+    {% if proxy_id %}
+    - name: proxy-ssh-keys
+      secret:
+        secretName: proxy-ssh-keys
+        defaultMode: 0600
+    {% endif %}
+
   {% if priorityClassName %}
   priorityClassName: {{ priorityClassName }}
   {% endif %}
@@ -73,10 +80,36 @@ spec:
         - --profile
         - "@{{ profile_filename }}"
       {%- endif %}
+      {% if proxy_id %}
+        - --proxyServer
+        - "ssh://{{ ssh_proxy_auth }}"
+        - --sshProxyPrivateKeyFile
+        - /tmp/ssh-proxy/private-key
+        - --sshProxyKnownHostsFile
+        - /tmp/ssh-proxy/known-hosts
+      {% endif %}
 
       volumeMounts:
         - name: crawler-workdir
           mountPath: /tmp/home
+      {% if proxy_id %}
+        - name: proxy-ssh-keys
+          mountPath: /tmp/ssh-proxy/private-key
+          subPath: {{ proxy_id }}-private-key
+          readOnly: true
+        - name: proxy-ssh-keys
+          mountPath: /tmp/ssh-proxy/known-hosts
+          subPath: {{ proxy_id }}-known-hosts
+          readOnly: true
+        - name: proxy-ssh-keys
+          mountPath: /etc/passwd
+          subPath: passwd
+          readOnly: true
+        - name: proxy-ssh-keys
+          mountPath: /etc/group
+          subPath: group
+          readOnly: true
+       {% endif %}
 
       envFrom:
         - secretRef:

chart/templates/proxies.yaml

@@ -12,7 +12,7 @@ stringData:
   {{ .id }}-private-key: |
 {{ .private_key | indent 4 }}
   {{ .id }}-known-hosts: |
-    {{ .host_public_key | nindent 4 }}
+{{ .host_public_key | indent 4 }}
 {{- end }}
 
   # slightly hacky: override /etc/passwd and /etc/group in crawler to be able to ssh to proxies

frontend/src/features/browser-profiles/new-browser-profile-dialog.ts

@@ -10,7 +10,8 @@ import {
 import queryString from "query-string";
 
 import type { Dialog } from "@/components/ui/dialog";
-import { type SelectCrawlerChangeEvent } from "@/components/ui/select-crawler";
+import type { SelectCrawlerChangeEvent } from "@/components/ui/select-crawler";
+import type { SelectCrawlerProxyChangeEvent } from "@/components/ui/select-crawler-proxy";
 import type { AuthState } from "@/utils/AuthService";
 import LiteElement, { html } from "@/utils/LiteElement";
 
@@ -32,6 +33,9 @@ export class NewBrowserProfileDialog extends LiteElement {
   @state()
   private crawlerChannel = "default";
 
+  @state()
+  private proxyId: string | null = null;
+
   @query("btrix-dialog")
   private readonly dialog?: Dialog;
 
@@ -88,6 +92,15 @@ export class NewBrowserProfileDialog extends LiteElement {
               (this.crawlerChannel = e.detail.value!)}
           ></btrix-select-crawler>
         </div>
+        <div class="mt-4">
+          <btrix-select-crawler-proxy
+            orgId=${this.orgId}
+            .proxyId="${this.proxyId || ""}"
+            .authState=${this.authState}
+            @on-change=${(e: SelectCrawlerProxyChangeEvent) =>
+              (this.proxyId = e.detail.value!)}
+          ></btrix-select-crawler-proxy>
+        </div>
         <input class="invisible size-0" type="submit" />
       </form>
       <div slot="footer" class="flex justify-between">
@@ -135,6 +148,7 @@ export class NewBrowserProfileDialog extends LiteElement {
       const data = await this.createBrowser({
         url: url,
         crawlerChannel: this.crawlerChannel,
+        proxyId: this.proxyId,
       });
 
       this.notify({
@@ -150,6 +164,7 @@ export class NewBrowserProfileDialog extends LiteElement {
           url,
           name: msg("My Profile"),
           crawlerChannel: this.crawlerChannel,
+          proxyId: this.proxyId,
         })}`,
       );
     } catch (e) {
@@ -165,13 +180,16 @@ export class NewBrowserProfileDialog extends LiteElement {
   private async createBrowser({
     url,
     crawlerChannel,
+    proxyId,
   }: {
     url: string;
     crawlerChannel: string;
+    proxyId: string | null;
   }) {
     const params = {
       url,
       crawlerChannel,
+      proxyId,
     };
 
     return this.apiFetch<{ browserid: string }>(

frontend/src/pages/org/browser-profiles-new.ts

@@ -43,9 +43,11 @@ export class BrowserProfilesNew extends TailwindElement {
     crawlerChannel?: string;
     profileId?: string | null;
     navigateUrl?: string;
+    proxyId: string | null;
   } = {
     name: "",
     url: "",
+    proxyId: null,
   };
 
   private readonly api = new APIController(this);
@@ -288,9 +290,11 @@ export class BrowserProfilesNew extends TailwindElement {
     }
 
     const crawlerChannel = this.browserParams.crawlerChannel || "default";
+    const proxyId = this.browserParams.proxyId;
     const data = await this.createBrowser({
       url,
       crawlerChannel,
+      proxyId,
     });
 
     this.nav.to(
@@ -300,6 +304,7 @@ export class BrowserProfilesNew extends TailwindElement {
         url,
         name: this.browserParams.name || msg("My Profile"),
         crawlerChannel,
+        proxyId,
       })}`,
     );
   }
@@ -314,6 +319,7 @@ export class BrowserProfilesNew extends TailwindElement {
       name: formData.get("name"),
       description: formData.get("description"),
       crawlerChannel: this.browserParams.crawlerChannel,
+      proxyId: this.browserParams.proxyId,
     };
 
     try {
@@ -362,13 +368,16 @@ export class BrowserProfilesNew extends TailwindElement {
   private async createBrowser({
     url,
     crawlerChannel,
+    proxyId,
   }: {
     url: string;
     crawlerChannel: string;
+    proxyId: string | null;
   }) {
     const params = {
       url,
       crawlerChannel,
+      proxyId,
     };
 
     return this.api.fetch<{ browserid: string }>(