diff --git a/internal/gatewayapi/clienttrafficpolicy.go b/internal/gatewayapi/clienttrafficpolicy.go
index b865817dae3..b27fce91aba 100644
--- a/internal/gatewayapi/clienttrafficpolicy.go
+++ b/internal/gatewayapi/clienttrafficpolicy.go
@@ -318,7 +318,10 @@ func (t *Translator) translateClientTrafficPolicyForListener(policy *egv1a1.Clie
 
 		// enable http3 if set and TLS is enabled
 		if httpIR.TLS != nil && policy.Spec.HTTP3 != nil {
-			httpIR.HTTP3 = &ir.HTTP3Settings{}
+			http3 := &ir.HTTP3Settings{
+				QUICPort: int32(l.Port),
+			}
+			httpIR.HTTP3 = http3
 			var proxyListenerIR *ir.ProxyListener
 			for _, proxyListener := range infraIR[irKey].Proxy.Listeners {
 				if proxyListener.Name == irListenerName {
@@ -327,7 +330,7 @@ func (t *Translator) translateClientTrafficPolicyForListener(policy *egv1a1.Clie
 				}
 			}
 			if proxyListenerIR != nil {
-				proxyListenerIR.HTTP3 = &ir.HTTP3Settings{}
+				proxyListenerIR.HTTP3 = http3
 			}
 		}
 
diff --git a/internal/gatewayapi/testdata/clienttrafficpolicy-http3.out.yaml b/internal/gatewayapi/testdata/clienttrafficpolicy-http3.out.yaml
index 867a2fa76a8..745aa593889 100644
--- a/internal/gatewayapi/testdata/clienttrafficpolicy-http3.out.yaml
+++ b/internal/gatewayapi/testdata/clienttrafficpolicy-http3.out.yaml
@@ -106,7 +106,8 @@ infraIR:
     proxy:
       listeners:
       - address: null
-        http3: {}
+        http3:
+          quicPort: 443
         name: envoy-gateway/gateway-1/tls
         ports:
         - containerPort: 10443
@@ -127,7 +128,8 @@ xdsIR:
     - address: 0.0.0.0
       hostnames:
       - '*'
-      http3: {}
+      http3:
+        quicPort: 443
       isHTTP2: false
       name: envoy-gateway/gateway-1/tls
       path:
diff --git a/internal/ir/infra.go b/internal/ir/infra.go
index 5c2a95505a8..60a26bddcc2 100644
--- a/internal/ir/infra.go
+++ b/internal/ir/infra.go
@@ -77,6 +77,7 @@ type ProxyListener struct {
 
 // HTTP3Settings provides HTTP/3 configuration on the listener.
 type HTTP3Settings struct {
+	QUICPort int32 `json:"quicPort" yaml:"quicPort"`
 }
 
 // ListenerPort defines a network port of a listener.
diff --git a/internal/xds/translator/testdata/in/xds-ir/http3.yaml b/internal/xds/translator/testdata/in/xds-ir/http3.yaml
index e2c16007855..299ea3241bb 100644
--- a/internal/xds/translator/testdata/in/xds-ir/http3.yaml
+++ b/internal/xds/translator/testdata/in/xds-ir/http3.yaml
@@ -1,6 +1,7 @@
 http:
 - address: 0.0.0.0
-  http3: {}
+  http3:
+    quicPort: 443
   hostnames:
   - '*'
   isHTTP2: false
diff --git a/internal/xds/translator/testdata/out/xds-ir/http3.routes.yaml b/internal/xds/translator/testdata/out/xds-ir/http3.routes.yaml
index 0680b361d15..f3e513cea47 100644
--- a/internal/xds/translator/testdata/out/xds-ir/http3.routes.yaml
+++ b/internal/xds/translator/testdata/out/xds-ir/http3.routes.yaml
@@ -12,7 +12,7 @@
       - append: true
         header:
           key: alt-svc
-          value: h3=":10443"; ma=86400
+          value: h3=":443"; ma=86400
       route:
         cluster: httproute/default/httproute-1/rule/0
         upgradeConfigs:
diff --git a/internal/xds/translator/translator.go b/internal/xds/translator/translator.go
index d77ed321502..d93361337eb 100644
--- a/internal/xds/translator/translator.go
+++ b/internal/xds/translator/translator.go
@@ -259,7 +259,7 @@ func (t *Translator) processHTTPListenerXdsTranslation(
 			}
 
 			if enabledHTTP3 {
-				http3AltSvcHeader := buildHTTP3AltSvcHeader(int(httpListener.Port))
+				http3AltSvcHeader := buildHTTP3AltSvcHeader(int(httpListener.HTTP3.QUICPort))
 				if xdsRoute.ResponseHeadersToAdd == nil {
 					xdsRoute.ResponseHeadersToAdd = make([]*corev3.HeaderValueOption, 0)
 				}