Skip to content

Commit

Permalink
Merge branch 'master' into chore.jobsdbPayloadColumnType
Browse files Browse the repository at this point in the history
  • Loading branch information
Sidddddarth authored Dec 19, 2024
2 parents 8524933 + 7425659 commit 37133be
Show file tree
Hide file tree
Showing 5 changed files with 19 additions and 25 deletions.
8 changes: 4 additions & 4 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -107,10 +107,10 @@ require (
golang.org/x/exp v0.0.0-20240909161429-701f63a606c0
golang.org/x/oauth2 v0.24.0
golang.org/x/sync v0.10.0
google.golang.org/api v0.211.0
google.golang.org/api v0.212.0
google.golang.org/genproto/googleapis/rpc v0.0.0-20241206012308-a4fef0638583
google.golang.org/grpc v1.68.1
google.golang.org/protobuf v1.35.2
google.golang.org/protobuf v1.36.0
)

require (
Expand All @@ -124,9 +124,9 @@ require (
require (
cel.dev/expr v0.16.1 // indirect
cloud.google.com/go v0.116.0 // indirect
cloud.google.com/go/auth v0.12.1 // indirect
cloud.google.com/go/auth v0.13.0 // indirect
cloud.google.com/go/auth/oauth2adapt v0.2.6 // indirect
cloud.google.com/go/compute/metadata v0.5.2 // indirect
cloud.google.com/go/compute/metadata v0.6.0 // indirect
cloud.google.com/go/iam v1.2.2 // indirect
cloud.google.com/go/monitoring v1.21.2 // indirect
dario.cat/mergo v1.0.1 // indirect
Expand Down
16 changes: 8 additions & 8 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -34,8 +34,8 @@ cloud.google.com/go v0.100.1/go.mod h1:fs4QogzfH5n2pBXBP9vRiU+eCny7lD2vmFZy79Iuw
cloud.google.com/go v0.100.2/go.mod h1:4Xra9TjzAeYHrl5+oeLlzbM2k3mjVhZh4UqTZ//w99A=
cloud.google.com/go v0.116.0 h1:B3fRrSDkLRt5qSHWe40ERJvhvnQwdZiHu0bJOpldweE=
cloud.google.com/go v0.116.0/go.mod h1:cEPSRWPzZEswwdr9BxE6ChEn01dWlTaF05LiC2Xs70U=
cloud.google.com/go/auth v0.12.1 h1:n2Bj25BUMM0nvE9D2XLTiImanwZhO3DkfWSYS/SAJP4=
cloud.google.com/go/auth v0.12.1/go.mod h1:BFMu+TNpF3DmvfBO9ClqTR/SiqVIm7LukKF9mbendF4=
cloud.google.com/go/auth v0.13.0 h1:8Fu8TZy167JkW8Tj3q7dIkr2v4cndv41ouecJx0PAHs=
cloud.google.com/go/auth v0.13.0/go.mod h1:COOjD9gwfKNKz+IIduatIhYJQIc0mG3H102r/EMxX6Q=
cloud.google.com/go/auth/oauth2adapt v0.2.6 h1:V6a6XDu2lTwPZWOawrAa9HUK+DB2zfJyTuciBG5hFkU=
cloud.google.com/go/auth/oauth2adapt v0.2.6/go.mod h1:AlmsELtlEBnaNTL7jCj8VQFLy6mbZv0s4Q7NGBeQ5E8=
cloud.google.com/go/bigquery v1.0.1/go.mod h1:i/xbL2UlR5RvWAURpBYZTtm/cXjCha9lbfbpx4poX+o=
Expand All @@ -50,8 +50,8 @@ cloud.google.com/go/compute v0.1.0/go.mod h1:GAesmwr110a34z04OlxYkATPBEfVhkymfTB
cloud.google.com/go/compute v1.2.0/go.mod h1:xlogom/6gr8RJGBe7nT2eGsQYAFUbbv8dbC29qE3Xmw=
cloud.google.com/go/compute v1.3.0/go.mod h1:cCZiE1NHEtai4wiufUhW8I8S1JKkAnhnQJWM7YD99wM=
cloud.google.com/go/compute v1.5.0/go.mod h1:9SMHyhJlzhlkJqrPAc839t2BZFTSk6Jdj6mkzQJeu0M=
cloud.google.com/go/compute/metadata v0.5.2 h1:UxK4uu/Tn+I3p2dYWTfiX4wva7aYlKixAHn3fyqngqo=
cloud.google.com/go/compute/metadata v0.5.2/go.mod h1:C66sj2AluDcIqakBq/M8lw8/ybHgOZqin2obFxa/E5k=
cloud.google.com/go/compute/metadata v0.6.0 h1:A6hENjEsCDtC1k8byVsgwvVcioamEHvZ4j01OwKxG9I=
cloud.google.com/go/compute/metadata v0.6.0/go.mod h1:FjyFAW1MW0C203CEOMDTu3Dk1FlqW3Rga40jzHL4hfg=
cloud.google.com/go/datacatalog v1.23.0 h1:9F2zIbWNNmtrSkPIyGRQNsIugG5VgVVFip6+tXSdWLg=
cloud.google.com/go/datacatalog v1.23.0/go.mod h1:9Wamq8TDfL2680Sav7q3zEhBJSPBrDxJU8WtPJ25dBM=
cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
Expand Down Expand Up @@ -1772,8 +1772,8 @@ google.golang.org/api v0.69.0/go.mod h1:boanBiw+h5c3s+tBPgEzLDRHfFLWV0qXxRHz3ws7
google.golang.org/api v0.70.0/go.mod h1:Bs4ZM2HGifEvXwd50TtW70ovgJffJYw2oRCOFU/SkfA=
google.golang.org/api v0.71.0/go.mod h1:4PyU6e6JogV1f9eA4voyrTY2batOLdgZ5qZ5HOCc4j8=
google.golang.org/api v0.74.0/go.mod h1:ZpfMZOVRMywNyvJFeqL9HRWBgAuRfSjJFpe9QtRRyDs=
google.golang.org/api v0.211.0 h1:IUpLjq09jxBSV1lACO33CGY3jsRcbctfGzhj+ZSE/Bg=
google.golang.org/api v0.211.0/go.mod h1:XOloB4MXFH4UTlQSGuNUxw0UT74qdENK8d6JNsXKLi0=
google.golang.org/api v0.212.0 h1:BcRj3MJfHF3FYD29rk7u9kuu1SyfGqfHcA0hSwKqkHg=
google.golang.org/api v0.212.0/go.mod h1:gICpLlpp12/E8mycRMzgy3SQ9cFh2XnVJ6vJi/kQbvI=
google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
google.golang.org/appengine v1.4.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
google.golang.org/appengine v1.5.0/go.mod h1:xpcJRLb0r/rnEns0DIKYYv+WjYCduHsrkT7/EB5XEv4=
Expand Down Expand Up @@ -1922,8 +1922,8 @@ google.golang.org/protobuf v1.26.0/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQ
google.golang.org/protobuf v1.27.1/go.mod h1:9q0QmTI4eRPtz6boOQmLYwt+qCgq0jsYwAQnmE0givc=
google.golang.org/protobuf v1.28.0/go.mod h1:HV8QOd/L58Z+nl8r43ehVNZIU/HEI6OcFqwMG9pJV4I=
google.golang.org/protobuf v1.34.1/go.mod h1:c6P6GXX6sHbq/GpV6MGZEdwhWPcYBgnhAHhKbcUYpos=
google.golang.org/protobuf v1.35.2 h1:8Ar7bF+apOIoThw1EdZl0p1oWvMqTHmpA2fRTyZO8io=
google.golang.org/protobuf v1.35.2/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
google.golang.org/protobuf v1.36.0 h1:mjIs9gYtt56AzC4ZaffQuh88TZurBGhIJMBZGSxNerQ=
google.golang.org/protobuf v1.36.0/go.mod h1:9fA7Ob0pmnwhb644+1+CVWFRbNajQ6iRojtC/QF5bRE=
gopkg.in/alexcesaro/statsd.v2 v2.0.0 h1:FXkZSCZIH17vLCO5sO2UucTHsH9pc+17F6pl3JVCwMc=
gopkg.in/alexcesaro/statsd.v2 v2.0.0/go.mod h1:i0ubccKGzBVNBpdGV5MocxyA/XlLUJzA7SLonnE4drU=
gopkg.in/cenkalti/backoff.v1 v1.1.0 h1:Arh75ttbsvlpVA7WtVpH4u9h6Zl46xuptxqLxPiSo4Y=
Expand Down
10 changes: 2 additions & 8 deletions warehouse/archive/archiver.go
Original file line number Diff line number Diff line change
Expand Up @@ -431,18 +431,12 @@ func (a *Archiver) getStagingFilesData(
FROM
%s
WHERE
source_id = $1
AND destination_id = $2
AND id >= $3
and id <= $4;`,
upload_id = $1;`,
pq.QuoteIdentifier(warehouseutils.WarehouseStagingFilesTable),
)

stagingFileRows, err := txn.QueryContext(ctx, stmt,
u.sourceID,
u.destID,
u.startStagingFileId,
u.endStagingFileId,
u.uploadID,
)
if err != nil {
return nil, fmt.Errorf("cannot query staging files data: %w", err)
Expand Down
4 changes: 2 additions & 2 deletions warehouse/archive/testdata/dump.sql
Original file line number Diff line number Diff line change
Expand Up @@ -43,13 +43,13 @@ VALUES
1, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', NULL
NOW(), '{}', 1
),
(
2, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
'{}', 'test-sourceID', 'test-destinationID',
'succeeded', 1, 231, NOW(), NOW(), NOW(),
NOW(), '{}', NULL
NOW(), '{}', 2
),
(
3, 'rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz',
Expand Down
6 changes: 3 additions & 3 deletions warehouse/archive/testdata/storage.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.1.unix_time.json.gz": "{\"id\":1,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.2.unix_time.json.gz": "{\"id\":2,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":null}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.1.unix_time.json.gz": "{\"id\":1,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":1}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.2.unix_time.json.gz": "{\"id\":2,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":2}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.3.unix_time.json.gz": "{\"id\":3,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":3}\n",
"test-prefix/wh-staging-files.test-sourceID.test-destinationID.4.unix_time.json.gz": "{\"id\":4,\"location\":\"rudder/rudder-warehouse-staging-logs/2EUralUySYUs7hgsdU1lFXRSm/2022-09-20/1663650685.2EUralsdsDyZjOKU1lFXRSm.eeadsb4-a066-42f4-a90b-460161378e1b.json.gz\",\"source_id\":\"test-sourceID\",\"destination_id\":\"test-destinationID\",\"schema\":{},\"error\":null,\"status\":\"succeeded\",\"first_event_at\":\"{{.Now}}\",\"last_event_at\":\"{{.Now}}\",\"total_events\":1,\"created_at\":\"{{.Now}}\",\"updated_at\":\"{{.Now}}\",\"metadata\":{},\"workspace_id\":\"1\",\"total_bytes\":231,\"upload_id\":4}\n"
}
}

0 comments on commit 37133be

Please sign in to comment.