Merge pull request #2151 from TheThingsNetwork/backport/v3.6.1

Release v3.6.1

.goreleaser.yml

@@ -141,7 +141,7 @@ brews:
     skip_upload: auto
     install: |
       bin.install "ttn-lw-cli"
-      libexec.install %w[ttn-lw-stack public doc]
+      libexec.install %w[ttn-lw-stack public]
       env = {
           :TTN_LW_HTTP_STATIC_SEARCH_PATH => libexec/"public"
       }
@@ -163,7 +163,6 @@ brews:
     skip_upload: auto
     install: |
       bin.install "ttn-lw-cli"
-      libexec.install doc
 
 blobs:
   - provider: s3

CHANGELOG.md

@@ -19,6 +19,34 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Security
 
+## [3.6.1] (2020-03-13)
+
+### Added
+
+- New `list` and `request-validation` subcommands for the CLI's `contact-info` commands.
+- Device Claim Authentication Code page in the Console.
+- Gateway Server rate limiting support for the UDP frontend, see (`--gs.udp.rate-limiting` options).
+- Uplink deduplication via Redis in Network Server.
+
+### Changed
+
+- Network and Application Servers now maintain application downlink queue per-session.
+- Gateway Server skips setting up an upstream if the DevAddr prefixes to forward are empty.
+- Gateway connection stats are now cached in Redis (see `--cache.service` and `--gs.update-connections-stats-debounce-time` options).
+
+### Fixed
+
+- Telemetry and events for gateway statuses.
+- Handling of downlink frame counters exceeding 65535.
+- Creating 1.0.4 ABP end devices via the Console.
+- ADR uplink handling.
+- Uplink retransmission handling.
+- Synchronizing Basic Station concentrator time after reconnect or initial connect after long inactivity.
+
+### Security
+
+- Changing username and password to be not required in pubsub integration.
+
 ## [3.6.0] (2020-02-27)
 
 ### Added
@@ -634,7 +662,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 NOTE: These links should respect backports. See https://github.com/TheThingsNetwork/lorawan-stack/pull/1444/files#r333379706.
 -->
 
-[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.6.0...HEAD
+[unreleased]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.6.1...HEAD
+[3.6.1]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.6.0...v3.6.1
 [3.6.0]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.5.3...v3.6.0
 [3.5.3]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.5.2...v3.5.3
 [3.5.2]: https://github.com/TheThingsNetwork/lorawan-stack/compare/v3.5.1...v3.5.2

api/api.md

@@ -1860,7 +1860,7 @@ SDKs are responsible for combining (if desired) the three.
 | `recent_adr_uplinks` | [`UplinkMessage`](#ttn.lorawan.v3.UplinkMessage) | repeated | Recent uplink messages with ADR bit set to 1 sorted by time. Stored in Network Server. The field is reset each time an uplink message carrying MACPayload is received with ADR bit set to 0. The number of messages stored is in the range [0,20]; |
 | `recent_uplinks` | [`UplinkMessage`](#ttn.lorawan.v3.UplinkMessage) | repeated | Recent uplink messages sorted by time. Stored in Network Server. The number of messages stored may depend on configuration. |
 | `recent_downlinks` | [`DownlinkMessage`](#ttn.lorawan.v3.DownlinkMessage) | repeated | Recent downlink messages sorted by time. Stored in Network Server. The number of messages stored may depend on configuration. |
-| `queued_application_downlinks` | [`ApplicationDownlink`](#ttn.lorawan.v3.ApplicationDownlink) | repeated | Queued Application downlink messages. Stored in Application Server, which sets them on the Network Server. |
+| `queued_application_downlinks` | [`ApplicationDownlink`](#ttn.lorawan.v3.ApplicationDownlink) | repeated | Queued Application downlink messages. Stored in Application Server, which sets them on the Network Server. This field is deprecated and is always set equal to session.queued_application_downlinks. |
 | `formatters` | [`MessagePayloadFormatters`](#ttn.lorawan.v3.MessagePayloadFormatters) |  | The payload formatters for this end device. Stored in Application Server. Copied on creation from template identified by version_ids. |
 | `provisioner_id` | [`string`](#string) |  | ID of the provisioner. Stored in Join Server. |
 | `provisioning_data` | [`google.protobuf.Struct`](#google.protobuf.Struct) |  | Vendor-specific provisioning data. Stored in Join Server. |
@@ -2260,6 +2260,7 @@ This is used internally by the Network Server and is read only.
 | `last_a_f_cnt_down` | [`uint32`](#uint32) |  | Last application downlink frame counter value used. Application Server only. |
 | `last_conf_f_cnt_down` | [`uint32`](#uint32) |  | Frame counter of the last confirmed downlink message sent. Network Server only. |
 | `started_at` | [`google.protobuf.Timestamp`](#google.protobuf.Timestamp) |  | Time when the session started. Network Server only. |
+| `queued_application_downlinks` | [`ApplicationDownlink`](#ttn.lorawan.v3.ApplicationDownlink) | repeated | Queued Application downlink messages. Stored in Application Server and Network Server. |
 
 #### Field Rules
 

api/api.swagger.json

@@ -9875,7 +9875,7 @@
           "items": {
             "$ref": "#/definitions/v3ApplicationDownlink"
           },
-          "description": "Queued Application downlink messages. Stored in Application Server,\nwhich sets them on the Network Server."
+          "description": "Queued Application downlink messages. Stored in Application Server,\nwhich sets them on the Network Server.\nThis field is deprecated and is always set equal to session.queued_application_downlinks."
         },
         "formatters": {
           "$ref": "#/definitions/v3MessagePayloadFormatters",
@@ -12035,6 +12035,13 @@
           "type": "string",
           "format": "date-time",
           "description": "Time when the session started. Network Server only."
+        },
+        "queued_application_downlinks": {
+          "type": "array",
+          "items": {
+            "$ref": "#/definitions/v3ApplicationDownlink"
+          },
+          "description": "Queued Application downlink messages. Stored in Application Server and Network Server."
         }
       }
     },

api/end_device.proto

@@ -50,6 +50,8 @@ message Session {
   uint32 last_conf_f_cnt_down = 7;
   // Time when the session started. Network Server only.
   google.protobuf.Timestamp started_at = 8 [(gogoproto.nullable) = false, (gogoproto.stdtime) = true];
+  // Queued Application downlink messages. Stored in Application Server and Network Server.
+  repeated ApplicationDownlink queued_application_downlinks = 9;
 }
 
 // MACParameters represent the parameters of the device's MAC layer (active or desired).
@@ -510,6 +512,7 @@ message EndDevice {
   repeated DownlinkMessage recent_downlinks = 39;
   // Queued Application downlink messages. Stored in Application Server,
   // which sets them on the Network Server.
+  // This field is deprecated and is always set equal to session.queued_application_downlinks.
   repeated ApplicationDownlink queued_application_downlinks = 40;
 
   // The payload formatters for this end device. Stored in Application Server.

api/identifiers.proto

@@ -76,6 +76,7 @@ message UserIdentifiers {
 // OrganizationOrUserIdentifiers contains either organization or user identifiers.
 message OrganizationOrUserIdentifiers {
   oneof ids {
+    option (validate.required) = true;
     OrganizationIdentifiers organization_ids = 1 [(gogoproto.customname) = "OrganizationIDs"];
     UserIdentifiers user_ids = 2 [(gogoproto.customname) = "UserIDs"];
   }
@@ -84,6 +85,7 @@ message OrganizationOrUserIdentifiers {
 // EntityIdentifiers contains one of the possible entity identifiers.
 message EntityIdentifiers {
   oneof ids {
+    option (validate.required) = true;
     ApplicationIdentifiers application_ids = 1 [(gogoproto.customname) = "ApplicationIDs"];
     ClientIdentifiers client_ids = 2 [(gogoproto.customname) = "ClientIDs"];
     EndDeviceIdentifiers device_ids = 3 [(gogoproto.customname) = "DeviceIDs"];

cmd/internal/shared/config.go

@@ -17,8 +17,10 @@ package shared
 import (
 	"time"
 
+	"go.thethings.network/lorawan-stack/pkg/cluster"
 	"go.thethings.network/lorawan-stack/pkg/config"
 	"go.thethings.network/lorawan-stack/pkg/log"
+	"go.thethings.network/lorawan-stack/pkg/redis"
 	"golang.org/x/crypto/acme"
 )
 
@@ -42,7 +44,7 @@ var DefaultTLSConfig = config.TLS{
 }
 
 // DefaultClusterConfig is the default cluster configuration.
-var DefaultClusterConfig = config.Cluster{}
+var DefaultClusterConfig = cluster.Config{}
 
 // DefaultHTTPConfig is the default HTTP config.
 var DefaultHTTPConfig = config.HTTP{
@@ -75,10 +77,10 @@ var DefaultGRPCConfig = config.GRPC{
 }
 
 // DefaultRedisConfig is the default config for Redis.
-var DefaultRedisConfig = config.Redis{
-	Address:   "localhost:6379",
-	Database:  0,
-	Namespace: []string{"ttn", "v3"},
+var DefaultRedisConfig = redis.Config{
+	Address:       "localhost:6379",
+	Database:      0,
+	RootNamespace: []string{"ttn", "v3"},
 }
 
 // DefaultEventsConfig is the default config for Events.

cmd/internal/shared/events.go

@@ -32,11 +32,7 @@ func InitializeEvents(ctx context.Context, config config.ServiceBase) (err error
 	case "internal":
 		return nil // this is the default.
 	case "redis":
-		if !config.Events.Redis.IsZero() {
-			events.SetDefaultPubSub(redis.NewPubSub(config.Events.Redis))
-		} else {
-			events.SetDefaultPubSub(redis.NewPubSub(config.Redis))
-		}
+		events.SetDefaultPubSub(redis.NewPubSub(config.Events.Redis))
 		return nil
 	case "cloud":
 		ps, err := cloud.NewPubSub(ctx, config.Events.Cloud.PublishURL, config.Events.Cloud.SubscribeURL)

cmd/internal/shared/gatewayserver/config.go

@@ -48,6 +48,7 @@ var DefaultGatewayServerConfig = gatewayserver.Config{
 		PublicAddress:    fmt.Sprintf("%s:1882", shared.DefaultPublicHost),
 		PublicTLSAddress: fmt.Sprintf("%s:8882", shared.DefaultPublicHost),
 	},
+	UpdateConnectionStatsDebounceTime: 3 * time.Second,
 	BasicStation: gatewayserver.BasicStationConfig{
 		Listen:         ":1887",
 		ListenTLS:      ":8887",

cmd/internal/shared/init.go

@@ -22,6 +22,15 @@ import (
 
 // Initialize global packages.
 func Initialize(ctx context.Context, config config.ServiceBase) error {
+	// Fallback to the default Redis configuration for the cache system
+	if config.Cache.Redis.IsZero() {
+		config.Cache.Redis = config.Redis
+	}
+	// Fallback to the default Redis configuration for the events system
+	if config.Events.Redis.IsZero() {
+		config.Events.Redis = config.Redis
+	}
+
 	if err := InitializeEvents(ctx, config); err != nil {
 		return err
 	}

cmd/ttn-lw-cli/commands/contact_info.go

@@ -27,6 +27,33 @@ import (
 	"go.thethings.network/lorawan-stack/pkg/ttnpb"
 )
 
+func listContactInfo(entityID *ttnpb.EntityIdentifiers) ([]*ttnpb.ContactInfo, error) {
+	is, err := api.Dial(ctx, config.IdentityServerGRPCAddress)
+	if err != nil {
+		return nil, err
+	}
+	fieldMask := types.FieldMask{Paths: []string{"contact_info"}}
+	var res interface{}
+	switch id := entityID.Identifiers().(type) {
+	case *ttnpb.ApplicationIdentifiers:
+		res, err = ttnpb.NewApplicationRegistryClient(is).Get(ctx, &ttnpb.GetApplicationRequest{ApplicationIdentifiers: *id, FieldMask: fieldMask})
+	case *ttnpb.ClientIdentifiers:
+		res, err = ttnpb.NewClientRegistryClient(is).Get(ctx, &ttnpb.GetClientRequest{ClientIdentifiers: *id, FieldMask: fieldMask})
+	case *ttnpb.GatewayIdentifiers:
+		res, err = ttnpb.NewGatewayRegistryClient(is).Get(ctx, &ttnpb.GetGatewayRequest{GatewayIdentifiers: *id, FieldMask: fieldMask})
+	case *ttnpb.OrganizationIdentifiers:
+		res, err = ttnpb.NewOrganizationRegistryClient(is).Get(ctx, &ttnpb.GetOrganizationRequest{OrganizationIdentifiers: *id, FieldMask: fieldMask})
+	case *ttnpb.UserIdentifiers:
+		res, err = ttnpb.NewUserRegistryClient(is).Get(ctx, &ttnpb.GetUserRequest{UserIdentifiers: *id, FieldMask: fieldMask})
+	default:
+		panic(fmt.Errorf("no contact info in %T", id))
+	}
+	if err != nil {
+		return nil, err
+	}
+	return res.(interface{ GetContactInfo() []*ttnpb.ContactInfo }).GetContactInfo(), nil
+}
+
 func updateContactInfo(entityID *ttnpb.EntityIdentifiers, updater func([]*ttnpb.ContactInfo) ([]*ttnpb.ContactInfo, error)) ([]*ttnpb.ContactInfo, error) {
 	is, err := api.Dial(ctx, config.IdentityServerGRPCAddress)
 	if err != nil {
@@ -118,6 +145,21 @@ func contactInfoCommands(entity string, getID func(cmd *cobra.Command, args []st
 		Use:   "contact-info",
 		Short: fmt.Sprintf("Manage %s contact info", entity),
 	}
+	list := &cobra.Command{
+		Use:     fmt.Sprintf("list [%s-id]", entity),
+		Aliases: []string{"ls", "get"},
+		RunE: func(cmd *cobra.Command, args []string) error {
+			id, err := getID(cmd, args)
+			if err != nil {
+				return err
+			}
+			contactInfo, err := listContactInfo(id)
+			if err != nil {
+				return err
+			}
+			return io.Write(os.Stdout, config.OutputFormat, contactInfo)
+		},
+	}
 	add := &cobra.Command{
 		Use: fmt.Sprintf("add [%s-id]", entity),
 		RunE: func(cmd *cobra.Command, args []string) error {
@@ -175,6 +217,25 @@ func contactInfoCommands(entity string, getID func(cmd *cobra.Command, args []st
 			return io.Write(os.Stdout, config.OutputFormat, updatedInfo)
 		},
 	}
+	requestValidation := &cobra.Command{
+		Use:   fmt.Sprintf("request-validation [%s-id]", entity),
+		Short: "Request validation for entity contact info",
+		RunE: func(cmd *cobra.Command, args []string) error {
+			id, err := getID(cmd, args)
+			if err != nil {
+				return err
+			}
+			is, err := api.Dial(ctx, config.IdentityServerGRPCAddress)
+			if err != nil {
+				return err
+			}
+			res, err := ttnpb.NewContactInfoRegistryClient(is).RequestValidation(ctx, id)
+			if err != nil {
+				return err
+			}
+			return io.Write(os.Stdout, config.OutputFormat, res)
+		},
+	}
 	validate := &cobra.Command{
 		Use:   "validate [reference] [token]",
 		Short: "Validate contact info",
@@ -212,8 +273,12 @@ func contactInfoCommands(entity string, getID func(cmd *cobra.Command, args []st
 	}
 	add.Flags().AddFlagSet(contactInfoFlags)
 	cmd.AddCommand(add)
+	list.Flags().AddFlagSet(contactInfoFlags)
+	cmd.AddCommand(list)
 	remove.Flags().AddFlagSet(contactInfoFlags)
 	cmd.AddCommand(remove)
+	requestValidation.Flags().AddFlagSet(contactInfoFlags)
+	cmd.AddCommand(requestValidation)
 	validate.Flags().String("reference", "", "Reference of the requested validation")
 	validate.Flags().String("token", "", "Token that you received")
 	cmd.AddCommand(validate)