diff --git a/.github/workflows/integration-tests-against-emulator.yaml b/.github/workflows/integration-tests-against-emulator.yaml index 63999ea5e..f39322639 100644 --- a/.github/workflows/integration-tests-against-emulator.yaml +++ b/.github/workflows/integration-tests-against-emulator.yaml @@ -114,21 +114,28 @@ jobs: - run: mysql -v -P 3306 --protocol=tcp -u root -proot < test_data/mysql_foreignkeyaction_dump.test.out # init sql server with test_data + # since we use ubuntu-latest container, we should ensure that the path matches the latest from https://packages.microsoft.com/config/ubuntu/ + # while its possible to infer the latest from the path in the run script, it will make the run section more complex and hard to maintian. - name: Install sqlcmd required for loading .sql files run: | curl https://packages.microsoft.com/keys/microsoft.asc | sudo apt-key add - - curl https://packages.microsoft.com/config/ubuntu/16.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list - sudo apt-get update - sudo apt-get install mssql-tools unixodbc-dev - echo 'export PATH="$PATH:/opt/mssql-tools/bin"' >> ~/.bash_profile - - run: sqlcmd -? - - run: sqlcmd -U sa -P ${MSSQL_SA_PASSWORD} -i test_data/sqlserver.test.out + curl https://packages.microsoft.com/config/ubuntu/24.04/prod.list | sudo tee /etc/apt/sources.list.d/msprod.list + sudo apt-get update + sudo ACCEPT_EULA=Y apt-get install -y msodbcsql18 + sudo apt-get install mssql-tools18 unixodbc-dev + set -x + ls /opt/mssql-tools18/bin/ + set +x + echo 'export PATH="$PATH:/opt/mssql-tools18/bin"' >> ~/.bashrc + - run: /opt/mssql-tools18/bin/sqlcmd -C -? + - run: /opt/mssql-tools18/bin/sqlcmd -U sa -P ${MSSQL_SA_PASSWORD} -i test_data/sqlserver.test.out -C # sqlplus set up init oracle db. - name: Install sqlplus required for loading .sql files run: | sudo apt-get update - sudo apt-get install -y libaio1 rpm2cpio cpio + sudo apt-get install -y libaio1t64 rpm2cpio cpio + sudo ln -s /usr/lib/x86_64-linux-gnu/libaio.so.1t64 /usr/lib/libaio.so.1 curl -O https://download.oracle.com/otn_software/linux/instantclient/2340000/oracle-instantclient-basic-23.4.0.24.05-1.el9.x86_64.rpm curl -O https://download.oracle.com/otn_software/linux/instantclient/2340000/oracle-instantclient-sqlplus-23.4.0.24.05-1.el9.x86_64.rpm rpm2cpio oracle-instantclient-basic-23.4.0.24.05-1.el9.x86_64.rpm | sudo cpio -idmv diff --git a/common/constants/constants.go b/common/constants/constants.go index 84c239e1c..b5cdb0246 100644 --- a/common/constants/constants.go +++ b/common/constants/constants.go @@ -123,7 +123,8 @@ const ( DLQ_GCS string = "dlq" //VerifyExpresions API - CHECK_EXPRESSION = "CHECK" + CHECK_EXPRESSION = "CHECK" DEFAUT_EXPRESSION = "DEFAULT" - + DEFAULT_GENERATED = "DEFAULT_GENERATED" + TEMP_DB = "smt-staging-db" ) diff --git a/conversion/conversion_from_source.go b/conversion/conversion_from_source.go index 0290af537..1eec13068 100644 --- a/conversion/conversion_from_source.go +++ b/conversion/conversion_from_source.go @@ -54,6 +54,9 @@ type DataFromSourceImpl struct{} func (sads *SchemaFromSourceImpl) schemaFromDatabase(migrationProjectId string, sourceProfile profiles.SourceProfile, targetProfile profiles.TargetProfile, getInfo GetInfoInterface, processSchema common.ProcessSchemaInterface) (*internal.Conv, error) { conv := internal.MakeConv() conv.SpDialect = targetProfile.Conn.Sp.Dialect + conv.SpProjectId = targetProfile.Conn.Sp.Project + conv.SpInstanceId = targetProfile.Conn.Sp.Instance + conv.Source = sourceProfile.Driver //handle fetching schema differently for sharded migrations, we only connect to the primary shard to //fetch the schema. We reuse the SourceProfileConnection object for this purpose. var infoSchema common.InfoSchema @@ -159,6 +162,9 @@ func (sads *DataFromSourceImpl) dataFromCSV(ctx context.Context, sourceProfile p return nil, fmt.Errorf("dbName is mandatory in target-profile for csv source") } conv.SpDialect = targetProfile.Conn.Sp.Dialect + conv.SpProjectId = targetProfile.Conn.Sp.Project + conv.SpInstanceId = targetProfile.Conn.Sp.Instance + conv.Source = sourceProfile.Driver dialect, err := targetProfile.FetchTargetDialect(ctx) if err != nil { return nil, fmt.Errorf("could not fetch dialect: %v", err) diff --git a/expressions_api/expression_verify.go b/expressions_api/expression_verify.go index 9caebce83..7e1f8be24 100644 --- a/expressions_api/expression_verify.go +++ b/expressions_api/expression_verify.go @@ -6,6 +6,7 @@ import ( "fmt" "sync" + spannerclient "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/clients/spanner/client" spanneraccessor "github.com/GoogleCloudPlatform/spanner-migration-tool/accessors/spanner" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/constants" "github.com/GoogleCloudPlatform/spanner-migration-tool/common/task" @@ -18,6 +19,7 @@ const THREAD_POOL = 500 type ExpressionVerificationAccessor interface { //Batch API which parallelizes expression verification calls VerifyExpressions(ctx context.Context, verifyExpressionsInput internal.VerifyExpressionsInput) internal.VerifyExpressionsOutput + RefreshSpannerClient(ctx context.Context, project string, instance string) error } type ExpressionVerificationAccessorImpl struct { @@ -25,15 +27,42 @@ type ExpressionVerificationAccessorImpl struct { } func NewExpressionVerificationAccessorImpl(ctx context.Context, project string, instance string) (*ExpressionVerificationAccessorImpl, error) { - spannerAccessor, err := spanneraccessor.NewSpannerAccessorClientImplWithSpannerClient(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", project, instance, "smt-staging-db")) - if err != nil { - return nil, err + var spannerAccessor *spanneraccessor.SpannerAccessorImpl + var err error + if project != "" && instance != "" { + spannerAccessor, err = spanneraccessor.NewSpannerAccessorClientImplWithSpannerClient(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", project, instance, constants.TEMP_DB)) + if err != nil { + return nil, err + } + } else { + spannerAccessor, err = spanneraccessor.NewSpannerAccessorClientImpl(ctx) + if err != nil { + return nil, err + } } return &ExpressionVerificationAccessorImpl{ SpannerAccessor: spannerAccessor, }, nil } +// APIs to verify and process Spanner DLL features such as Default Values, Check Constraints +type DDLVerifier interface { + VerifySpannerDDL(conv *internal.Conv, expressionDetails []internal.ExpressionDetail) (internal.VerifyExpressionsOutput, error) + GetSourceExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail + GetSpannerExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail + RefreshSpannerClient(ctx context.Context, project string, instance string) error +} +type DDLVerifierImpl struct { + Expressions ExpressionVerificationAccessor +} + +func NewDDLVerifierImpl(ctx context.Context, project string, instance string) (*DDLVerifierImpl, error) { + expVerifier, err := NewExpressionVerificationAccessorImpl(ctx, project, instance) + return &DDLVerifierImpl{ + Expressions: expVerifier, + }, err +} + func (ev *ExpressionVerificationAccessorImpl) VerifyExpressions(ctx context.Context, verifyExpressionsInput internal.VerifyExpressionsInput) internal.VerifyExpressionsOutput { err := ev.validateRequest(verifyExpressionsInput) if err != nil { @@ -79,6 +108,15 @@ func (ev *ExpressionVerificationAccessorImpl) VerifyExpressions(ctx context.Cont return verifyExpressionsOutput } +func (ev *ExpressionVerificationAccessorImpl) RefreshSpannerClient(ctx context.Context, project string, instance string) error { + spannerClient, err := spannerclient.NewSpannerClientImpl(ctx, fmt.Sprintf("projects/%s/instances/%s/databases/%s", project, instance, constants.TEMP_DB)) + if err != nil { + return err + } + ev.SpannerAccessor.SpannerClient = spannerClient + return nil +} + func (ev *ExpressionVerificationAccessorImpl) verifyExpressionInternal(expressionDetail internal.ExpressionDetail, mutex *sync.Mutex) task.TaskResult[internal.ExpressionVerificationOutput] { var sqlStatement string switch expressionDetail.Type { @@ -129,3 +167,67 @@ func (ev *ExpressionVerificationAccessorImpl) removeExpressions(inputConv *inter } return convCopy, nil } + +func (ddlv *DDLVerifierImpl) VerifySpannerDDL(conv *internal.Conv, expressionDetails []internal.ExpressionDetail) (internal.VerifyExpressionsOutput, error) { + ctx := context.Background() + verifyExpressionsInput := internal.VerifyExpressionsInput{ + Conv: conv, + Source: conv.Source, + ExpressionDetailList: expressionDetails, + } + verificationResults := ddlv.Expressions.VerifyExpressions(ctx, verifyExpressionsInput) + + return verificationResults, verificationResults.Err +} + +func (ddlv *DDLVerifierImpl) GetSourceExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail { + expressionDetails := []internal.ExpressionDetail{} + // Collect default values for verification + for _, tableId := range tableIds { + srcTable := conv.SrcSchema[tableId] + for _, srcColId := range srcTable.ColIds { + srcCol := srcTable.ColDefs[srcColId] + if srcCol.DefaultValue.IsPresent { + defaultValueExp := internal.ExpressionDetail{ + ReferenceElement: internal.ReferenceElement{ + Name: conv.SpSchema[tableId].ColDefs[srcColId].T.Name, + }, + ExpressionId: srcCol.DefaultValue.Value.ExpressionId, + Expression: srcCol.DefaultValue.Value.Statement, + Type: "DEFAULT", + Metadata: map[string]string{"TableId": tableId, "ColId": srcColId}, + } + expressionDetails = append(expressionDetails, defaultValueExp) + } + } + } + return expressionDetails +} + +func (ddlv *DDLVerifierImpl) GetSpannerExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail { + expressionDetails := []internal.ExpressionDetail{} + // Collect default values for verification + for _, tableId := range tableIds { + spTable := conv.SpSchema[tableId] + for _, spColId := range spTable.ColIds { + spCol := spTable.ColDefs[spColId] + if spCol.DefaultValue.IsPresent { + defaultValueExp := internal.ExpressionDetail{ + ReferenceElement: internal.ReferenceElement{ + Name: conv.SpSchema[tableId].ColDefs[spColId].T.Name, + }, + ExpressionId: spCol.DefaultValue.Value.ExpressionId, + Expression: spCol.DefaultValue.Value.Statement, + Type: "DEFAULT", + Metadata: map[string]string{"TableId": tableId, "ColId": spColId}, + } + expressionDetails = append(expressionDetails, defaultValueExp) + } + } + } + return expressionDetails +} + +func (ddlv *DDLVerifierImpl) RefreshSpannerClient(ctx context.Context, project string, instance string) error { + return ddlv.Expressions.RefreshSpannerClient(ctx, project, instance) +} diff --git a/expressions_api/expression_verify_test.go b/expressions_api/expression_verify_test.go index 8dadb01f8..11f3facc2 100644 --- a/expressions_api/expression_verify_test.go +++ b/expressions_api/expression_verify_test.go @@ -17,6 +17,8 @@ import ( "github.com/GoogleCloudPlatform/spanner-migration-tool/expressions_api" "github.com/GoogleCloudPlatform/spanner-migration-tool/internal" "github.com/GoogleCloudPlatform/spanner-migration-tool/logger" + "github.com/GoogleCloudPlatform/spanner-migration-tool/schema" + "github.com/GoogleCloudPlatform/spanner-migration-tool/spanner/ddl" "github.com/googleapis/gax-go/v2" "github.com/stretchr/testify/assert" "go.uber.org/zap" @@ -32,8 +34,8 @@ func TestVerifyExpressions(t *testing.T) { conv := internal.MakeConv() ReadSessionFile(conv, "../../test_data/session_expression_verify.json") input := internal.VerifyExpressionsInput{ - Conv: conv, - Source: "mysql", + Conv: conv, + Source: "mysql", ExpressionDetailList: []internal.ExpressionDetail{ { Expression: "id > 10", @@ -297,3 +299,184 @@ func ReadSessionFile(conv *internal.Conv, sessionJSON string) error { } return nil } + +func TestVerifySpannerDDL(t *testing.T) { + conv := *internal.MakeConv() + testCases := []struct { + name string + conv internal.Conv + expressionDetails []internal.ExpressionDetail + verifyExpressionMock expressions_api.MockExpressionVerificationAccessor + errorExpected bool + }{ + { + name: "no error flow", + conv: conv, + expressionDetails: []internal.ExpressionDetail{}, + verifyExpressionMock: expressions_api.MockExpressionVerificationAccessor{ + VerifyExpressionsMock: func(ctx context.Context, verifyExpressionsInput internal.VerifyExpressionsInput) internal.VerifyExpressionsOutput { + return internal.VerifyExpressionsOutput{ + ExpressionVerificationOutputList: []internal.ExpressionVerificationOutput{}, + Err: nil, + } + }, + }, + errorExpected: false, + }, + { + name: "error flow", + conv: conv, + expressionDetails: []internal.ExpressionDetail{}, + verifyExpressionMock: expressions_api.MockExpressionVerificationAccessor{ + VerifyExpressionsMock: func(ctx context.Context, verifyExpressionsInput internal.VerifyExpressionsInput) internal.VerifyExpressionsOutput { + return internal.VerifyExpressionsOutput{ + ExpressionVerificationOutputList: []internal.ExpressionVerificationOutput{}, + Err: fmt.Errorf("error"), + } + }, + }, + errorExpected: true, + }, + } + + for _, tc := range testCases { + ddlV := expressions_api.DDLVerifierImpl{ + Expressions: &tc.verifyExpressionMock, + } + _, err := ddlV.VerifySpannerDDL(&tc.conv, tc.expressionDetails) + assert.Equal(t, tc.errorExpected, err != nil) + } +} + +func TestGetSourceExpressionDetails(t *testing.T) { + conv := internal.MakeConv() + conv.SrcSchema = map[string]schema.Table{ + "table1": { + ColIds: []string{"col1", "col2"}, + ColDefs: map[string]schema.Column{ + "col1": { + DefaultValue: ddl.DefaultValue{ + IsPresent: true, + Value: ddl.Expression{ + ExpressionId: "expr1", + Statement: "SELECT 1", + }, + }, + }, + "col2": { + DefaultValue: ddl.DefaultValue{}, + }, + }, + }, + } + conv.SpSchema = ddl.Schema{ + "table1": { + ColDefs: map[string]ddl.ColumnDef{ + "col1": { + T: ddl.Type{ + Name: "INT64", + }, + }, + }, + }, + } + + testCases := []struct { + name string + conv *internal.Conv + tableIds []string + expectedDetails []internal.ExpressionDetail + }{ + { + name: "single table with default value", + conv: conv, + tableIds: []string{"table1"}, + expectedDetails: []internal.ExpressionDetail{ + { + ReferenceElement: internal.ReferenceElement{ + Name: "INT64", + }, + ExpressionId: "expr1", + Expression: "SELECT 1", + Type: "DEFAULT", + Metadata: map[string]string{"TableId": "table1", "ColId": "col1"}, + }, + }, + }, + { + name: "no tables", + conv: conv, + tableIds: []string{}, + expectedDetails: []internal.ExpressionDetail{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ddlv := &expressions_api.DDLVerifierImpl{} + actualDetails := ddlv.GetSourceExpressionDetails(tc.conv, tc.tableIds) + assert.Equal(t, tc.expectedDetails, actualDetails) + }) + } +} + +func TestGetSpannerExpressionDetails(t *testing.T) { + conv := internal.MakeConv() + conv.SpSchema = ddl.Schema{ + "table1": { + ColIds: []string{"col1", "col2"}, + ColDefs: map[string]ddl.ColumnDef{ + "col1": { + DefaultValue: ddl.DefaultValue{ + IsPresent: true, + Value: ddl.Expression{ + ExpressionId: "expr1", + Statement: "SELECT 1", + }, + }, + }, + "col2": { + DefaultValue: ddl.DefaultValue{}, + }, + }, + }, + } + + testCases := []struct { + name string + conv *internal.Conv + tableIds []string + expectedDetails []internal.ExpressionDetail + }{ + { + name: "single table with default value", + conv: conv, + tableIds: []string{"table1"}, + expectedDetails: []internal.ExpressionDetail{ + { + ReferenceElement: internal.ReferenceElement{ + Name: conv.SpSchema["table1"].ColDefs["col1"].T.Name, + }, + ExpressionId: "expr1", + Expression: "SELECT 1", + Type: "DEFAULT", + Metadata: map[string]string{"TableId": "table1", "ColId": "col1"}, + }, + }, + }, + { + name: "no tables", + conv: conv, + tableIds: []string{}, + expectedDetails: []internal.ExpressionDetail{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + ddlv := &expressions_api.DDLVerifierImpl{} + actualDetails := ddlv.GetSpannerExpressionDetails(tc.conv, tc.tableIds) + assert.Equal(t, tc.expectedDetails, actualDetails) + }) + } +} diff --git a/expressions_api/mocks.go b/expressions_api/mocks.go new file mode 100644 index 000000000..b56e87060 --- /dev/null +++ b/expressions_api/mocks.go @@ -0,0 +1,69 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expressions_api + +import ( + "context" + + "github.com/GoogleCloudPlatform/spanner-migration-tool/internal" +) + +type MockExpressionVerificationAccessor struct { + VerifyExpressionsMock func(ctx context.Context, verifyExpressionsInput internal.VerifyExpressionsInput) internal.VerifyExpressionsOutput + RefreshSpannerClientMock func(ctx context.Context, project string, instance string) error +} + +func (mev *MockExpressionVerificationAccessor) VerifyExpressions(ctx context.Context, verifyExpressionsInput internal.VerifyExpressionsInput) internal.VerifyExpressionsOutput { + return mev.VerifyExpressionsMock(ctx, verifyExpressionsInput) +} + +func (mev *MockExpressionVerificationAccessor) RefreshSpannerClient(ctx context.Context, project string, instance string) error { + return mev.RefreshSpannerClientMock(ctx, project, instance) +} + +type MockDDLVerifier struct { + VerifySpannerDDLMock func(conv *internal.Conv, expressionDetails []internal.ExpressionDetail) (internal.VerifyExpressionsOutput, error) + GetSpannerExpressionDetailsMock func(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail + GetSourceExpressionDetailsMock func(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail + RefreshSpannerClientMock func(ctx context.Context, project string, instance string) error +} + +func (m *MockDDLVerifier) VerifySpannerDDL(conv *internal.Conv, expressionDetails []internal.ExpressionDetail) (internal.VerifyExpressionsOutput, error) { + if m.VerifySpannerDDLMock != nil { + return m.VerifySpannerDDLMock(conv, expressionDetails) + } + return internal.VerifyExpressionsOutput{}, nil +} + +func (m *MockDDLVerifier) GetSpannerExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail { + if m.GetSpannerExpressionDetailsMock != nil { + return m.GetSpannerExpressionDetailsMock(conv, tableIds) + } + return []internal.ExpressionDetail{} +} + +func (m *MockDDLVerifier) GetSourceExpressionDetails(conv *internal.Conv, tableIds []string) []internal.ExpressionDetail { + if m.GetSourceExpressionDetailsMock != nil { + return m.GetSourceExpressionDetailsMock(conv, tableIds) + } + return []internal.ExpressionDetail{} +} + +func (m *MockDDLVerifier) RefreshSpannerClient(ctx context.Context, project string, instance string) error { + if m.RefreshSpannerClientMock != nil { + return m.RefreshSpannerClientMock(ctx, project, instance) + } + return nil +} diff --git a/go.mod b/go.mod index b27aa33f9..4353254fa 100644 --- a/go.mod +++ b/go.mod @@ -31,9 +31,9 @@ require ( github.com/stretchr/testify v1.9.0 go.uber.org/ratelimit v0.3.1 go.uber.org/zap v1.23.0 - golang.org/x/crypto v0.22.0 + golang.org/x/crypto v0.31.0 golang.org/x/exp v0.0.0-20221023144134-a1e5550cf13e - golang.org/x/net v0.24.0 + golang.org/x/net v0.33.0 google.golang.org/api v0.178.0 google.golang.org/genproto v0.0.0-20240401170217-c3f982113cda google.golang.org/grpc v1.63.2 @@ -125,10 +125,10 @@ require ( go.uber.org/atomic v1.10.0 // indirect go.uber.org/multierr v1.8.0 // indirect golang.org/x/oauth2 v0.20.0 // indirect - golang.org/x/sync v0.7.0 // indirect - golang.org/x/sys v0.19.0 // indirect - golang.org/x/term v0.19.0 // indirect - golang.org/x/text v0.14.0 // indirect + golang.org/x/sync v0.10.0 // indirect + golang.org/x/sys v0.28.0 // indirect + golang.org/x/term v0.27.0 // indirect + golang.org/x/text v0.21.0 // indirect golang.org/x/xerrors v0.0.0-20231012003039-104605ab7028 // indirect google.golang.org/genproto/googleapis/api v0.0.0-20240506185236-b8a5c65736ae // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20240506185236-b8a5c65736ae // indirect diff --git a/go.sum b/go.sum index 89f4090a6..2603b49ad 100644 --- a/go.sum +++ b/go.sum @@ -664,6 +664,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.22.0 h1:g1v0xeRhjcugydODzvb3mEM9SQ0HGp9s/nh3COQ/C30= golang.org/x/crypto v0.22.0/go.mod h1:vr6Su+7cTlO45qkww3VDJlzDn0ctJvRgYbC2NvXHt+M= +golang.org/x/crypto v0.31.0 h1:ihbySMvVjLAeSH1IbfcRTkD/iNscyz8rGzjF/E5hV6U= +golang.org/x/crypto v0.31.0/go.mod h1:kDsLvtWBEx7MV9tJOj9bnXsPbxwJQ6csT/x4KIN4Ssk= golang.org/x/exp v0.0.0-20180321215751-8460e604b9de/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= @@ -744,6 +746,8 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.1.0/go.mod h1:Cx3nUiGt4eDBEyega/BKRp+/AlGL8hYe7U9odMt2Cco= golang.org/x/net v0.24.0 h1:1PcaxkF854Fu3+lvBIx5SYn9wRlBzzcnHZSiaFFAb0w= golang.org/x/net v0.24.0/go.mod h1:2Q7sJY5mzlzWjKtYUEXSlBWCdyaioyXzRB2RtU8KVE8= +golang.org/x/net v0.33.0 h1:74SYHlV8BIgHIFC/LrYkOGIwL19eTYXQ5wc6TBuO36I= +golang.org/x/net v0.33.0/go.mod h1:HXLR5J+9DxmrqMwG9qjGCxZ+zKXxBru04zlTvWlWuN4= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20190604053449-0f29369cfe45/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= @@ -767,6 +771,8 @@ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.7.0 h1:YsImfSBoP9QPYL0xyKJPq0gcaJdG3rInoqxTWbfQu9M= golang.org/x/sync v0.7.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= +golang.org/x/sync v0.10.0 h1:3NQrjDixjgGwUOCaF8w2+VYHv0Ve/vGYSbdkTa98gmQ= +golang.org/x/sync v0.10.0/go.mod h1:Czt+wKu1gCyEFDUtn0jG5QVvpJ6rzVqr5aXyt9drQfk= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180909124046-d0be0721c37e/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -826,11 +832,15 @@ golang.org/x/sys v0.0.0-20220722155257-8c9f86f7a55f/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.1.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.19.0 h1:q5f1RH2jigJ1MoAWp2KTp3gm5zAGFUTarQZ5U386+4o= golang.org/x/sys v0.19.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= +golang.org/x/sys v0.28.0 h1:Fksou7UEQUWlKvIdsqzJmUmCX3cZuD2+P3XyyzwMhlA= +golang.org/x/sys v0.28.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.1.0/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.19.0 h1:+ThwsDv+tYfnJFhF4L8jITxu1tdTWRTZpdsWgEgjL6Q= golang.org/x/term v0.19.0/go.mod h1:2CuTdWZ7KHSQwUzKva0cbMg6q2DMI3Mmxp+gKJbskEk= +golang.org/x/term v0.27.0 h1:WP60Sv1nlK1T6SupCHbXzSaN0b9wUmsPoRS9b61A23Q= +golang.org/x/term v0.27.0/go.mod h1:iMsnZpn0cago0GOrHO2+Y7u7JPn5AylBrcoWkElMTSM= golang.org/x/text v0.0.0-20170915032832-14c0d48ead0c/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.0/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= golang.org/x/text v0.3.1-0.20180807135948-17ff2d5776d2/go.mod h1:NqM8EUOU14njkJ3fqMW+pc6Ldnwhi/IjpwHt7yyuwOQ= @@ -842,6 +852,8 @@ golang.org/x/text v0.3.8/go.mod h1:E6s5w1FMmriuDzIBO73fBruAKo1PCIq6d2Q6DHfQ8WQ= golang.org/x/text v0.4.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8= golang.org/x/text v0.14.0 h1:ScX5w1eTa3QqT8oi6+ziP7dTV1S2+ALU0bI+0zXKWiQ= golang.org/x/text v0.14.0/go.mod h1:18ZOQIKpY8NJVqYksKHtTdi31H5itFRjB5/qKTNYzSU= +golang.org/x/text v0.21.0 h1:zyQAAkrwaneQ066sspRyJaG9VNi/YJ1NfzcGB3hZ/qo= +golang.org/x/text v0.21.0/go.mod h1:4IBbMaMmOPCJ8SecivzSH54+73PCFmPWxNTLm+vZkEQ= golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= golang.org/x/time v0.0.0-20191024005414-555d28b269f0/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ= diff --git a/internal/convert.go b/internal/convert.go index 62b8f30c6..413691260 100644 --- a/internal/convert.go +++ b/internal/convert.go @@ -54,6 +54,9 @@ type Conv struct { UI bool // Flag if UI interface was used for migration. ToDo: Remove flag after resource generation is introduced to UI SpSequences map[string]ddl.Sequence // Maps Spanner Sequences to Sequence Schema SrcSequences map[string]ddl.Sequence // Maps source-DB Sequences to Sequence schema information + SpProjectId string // Spanner Project Id + SpInstanceId string // Spanner Instance Id + Source string // Source Database type being migrated } type TableIssues struct { @@ -128,6 +131,7 @@ const ( SequenceCreated ForeignKeyActionNotSupported NumericPKNotSupported + DefaultValueError ) const ( @@ -291,17 +295,17 @@ type TableDetails struct { } type VerifyExpressionsInput struct { - Conv *Conv - Source string + Conv *Conv + Source string ExpressionDetailList []ExpressionDetail } type ExpressionDetail struct { ReferenceElement ReferenceElement - ExpressionId string - Expression string - Type string - Metadata map[string]string + ExpressionId string + Expression string + Type string + Metadata map[string]string } type ReferenceElement struct { @@ -310,13 +314,13 @@ type ReferenceElement struct { type ExpressionVerificationOutput struct { ExpressionDetail ExpressionDetail - Result bool - Err error + Result bool + Err error } type VerifyExpressionsOutput struct { ExpressionVerificationOutputList []ExpressionVerificationOutput - Err error + Err error } // MakeConv returns a default-configured Conv. diff --git a/internal/helpers.go b/internal/helpers.go index 8690dd10c..621b36365 100644 --- a/internal/helpers.go +++ b/internal/helpers.go @@ -26,7 +26,7 @@ import ( type Counter struct { counterMutex sync.Mutex - ObjectId string + ObjectId string } var Cntr Counter @@ -71,6 +71,9 @@ func GenerateRuleId() string { func GenerateSequenceId() string { return GenerateId("s") } +func GenerateExpressionId() string { + return GenerateId("e") +} func GetSrcColNameIdMap(srcs schema.Table) map[string]string { if len(srcs.ColNameIdMap) > 0 { diff --git a/internal/reports/report_helpers.go b/internal/reports/report_helpers.go index a066f0cde..38df589af 100644 --- a/internal/reports/report_helpers.go +++ b/internal/reports/report_helpers.go @@ -403,6 +403,12 @@ func buildTableReportBody(conv *internal.Conv, tableId string, issues map[string Description: fmt.Sprintf("UNIQUE constraint on column(s) '%s' replaced with primary key since table '%s' didn't have one. Spanner requires a primary key for every table", strings.Join(uniquePK, ", "), conv.SpSchema[tableId].Name), } l = append(l, toAppend) + case internal.DefaultValueError: + toAppend := Issue{ + Category: IssueDB[i].Category, + Description: fmt.Sprintf("%s for table '%s' column '%s'", IssueDB[i].Brief, conv.SpSchema[tableId].Name, spColName), + } + l = append(l, toAppend) default: toAppend := Issue{ Category: IssueDB[i].Category, @@ -562,6 +568,7 @@ var IssueDB = map[internal.SchemaIssue]struct { internal.ForeignKeyOnUpdate: {Brief: "Spanner supports only ON UPDATE NO ACTION", Severity: warning, Category: "FOREIGN_KEY_ACTIONS"}, internal.ForeignKeyActionNotSupported: {Brief: "Spanner supports foreign key action migration only for MySQL and PostgreSQL", Severity: warning, Category: "FOREIGN_KEY_ACTIONS"}, internal.NumericPKNotSupported: {Brief: "Spanner PostgreSQL does not support numeric primary keys / unique indices", Severity: warning, Category: "NUMERIC_PK_NOT_SUPPORTED"}, + internal.DefaultValueError: {Brief: "Some columns have default value expressions not supported by Spanner. Please fix them to continue migration.", Severity: Errors, batch: true, Category: "INCOMPATIBLE_DEFAULT_VALUE_CONSTRAINTS"}, } type Severity int diff --git a/schema/schema.go b/schema/schema.go index 48b125bba..7d73cd799 100644 --- a/schema/schema.go +++ b/schema/schema.go @@ -49,12 +49,13 @@ type Table struct { // Column represents a database column. // TODO: add support for foreign keys. type Column struct { - Name string - Type Type - NotNull bool - Ignored Ignored - Id string - AutoGen ddl.AutoGenCol + Name string + Type Type + NotNull bool + Ignored Ignored + Id string + AutoGen ddl.AutoGenCol + DefaultValue ddl.DefaultValue } // ForeignKey represents a foreign key. diff --git a/sources/common/infoschema.go b/sources/common/infoschema.go index 45bdb73fd..b4f9c9e7c 100644 --- a/sources/common/infoschema.go +++ b/sources/common/infoschema.go @@ -17,6 +17,7 @@ package common import ( "context" "fmt" + "strings" "sync" sp "cloud.google.com/go/spanner" @@ -253,3 +254,24 @@ func (is *InfoSchemaImpl) GetIncludedSrcTablesFromConv(conv *internal.Conv) (sch } return schemaToTablesMap, nil } + +// SanitizeDefaultValue removes extra characters added to Default Value in information schema in MySQL. +func SanitizeDefaultValue(defaultValue string, ty string, generated bool) string { + types := []string{"char", "varchar", "text", "varbinary", "tinyblob", "tinytext", "text", + "blob", "mediumtext", "mediumblob", "longtext", "longblob", "STRING"} + // Check if ty exists in the types array + stringType := false + for _, t := range types { + if t == ty { + stringType = true + break + } + } + defaultValue = strings.ReplaceAll(defaultValue, "_utf8mb4", "") + defaultValue = strings.ReplaceAll(defaultValue, "\\\\", "\\") + defaultValue = strings.ReplaceAll(defaultValue, "\\'", "'") + if !generated && stringType && !strings.HasPrefix(defaultValue, "'") && !strings.HasSuffix(defaultValue, "'") { + defaultValue = "'" + defaultValue + "'" + } + return defaultValue +} diff --git a/sources/common/infoschema_test.go b/sources/common/infoschema_test.go new file mode 100644 index 000000000..0286823aa --- /dev/null +++ b/sources/common/infoschema_test.go @@ -0,0 +1,47 @@ +// Copyright 2024 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package common + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + + +func TestSanitizeDefaultValue(t *testing.T) { + tests := []struct { + inputString string + ty string + generated bool + expectedString string + }{ + {"a", "char", true, "a"}, + {"b", "char", false, "'b'"}, + {"c", "int", true, "c"}, + {"d", "int", false, "d"}, + {"_utf8mb4\\'hello world\\'", "char", false, "'hello world'"}, + {"week(_utf8mb4\\'2024-06-20\\',0)", "char", true, "week('2024-06-20',0)"}, + {"_utf8mb4\\'This is a message \\\\nwith a newline\\\\rand a carriage return.\\'", "char", false, "'This is a message \\nwith a newline\\rand a carriage return.'"}, + {"strcmp(_utf8mb4\\'abc\\',_utf8mb4\\'abcd\\')", "char", true, "strcmp('abc','abcd')"}, + {"_utf8mb4\\'John\\\\\\'s Jack\\'", "char", false, "'John\\'s Jack'"}, + {"_utf8mb4\\'This product has\tmultiple features.\\'", "char", false, "'This product has\tmultiple features.'"}, + {"_utf8mb4\\'C:\\\\\\\\Users\\\\\\\\johndoe\\\\\\\\Documents\\\\\\\\myfile.txt\\'", "char", false, "'C:\\\\Users\\\\johndoe\\\\Documents\\\\myfile.txt'"}, + } + for _, test := range tests { + result := SanitizeDefaultValue(test.inputString, test.ty, test.generated) + assert.Equal(t, test.expectedString, result) + } +} diff --git a/sources/common/toddl.go b/sources/common/toddl.go index 16f4cf3ce..1b706274e 100644 --- a/sources/common/toddl.go +++ b/sources/common/toddl.go @@ -374,3 +374,32 @@ func CvtIndexHelper(conv *internal.Conv, tableId string, srcIndex schema.Index, } return spIndex } + +// Applies all valid expressions which can be migrated to spanner conv object +func spannerSchemaApplyExpressions(conv *internal.Conv, expressions internal.VerifyExpressionsOutput) { + for _, expression := range expressions.ExpressionVerificationOutputList { + switch expression.ExpressionDetail.Type { + case "DEFAULT": + { + tableId := expression.ExpressionDetail.Metadata["TableId"] + columnId := expression.ExpressionDetail.Metadata["ColId"] + + if expression.Result { + col := conv.SpSchema[tableId].ColDefs[columnId] + col.DefaultValue = ddl.DefaultValue{ + IsPresent: true, + Value: ddl.Expression{ + ExpressionId: expression.ExpressionDetail.ExpressionId, + Statement: expression.ExpressionDetail.Expression, + }, + } + conv.SpSchema[tableId].ColDefs[columnId] = col + } else { + colIssues := conv.SchemaIssues[tableId].ColumnLevelIssues[columnId] + colIssues = append(colIssues, internal.DefaultValue) + conv.SchemaIssues[tableId].ColumnLevelIssues[columnId] = colIssues + } + } + } + } +} diff --git a/sources/common/toddl_test.go b/sources/common/toddl_test.go index 63ecad71b..dcf5b3651 100644 --- a/sources/common/toddl_test.go +++ b/sources/common/toddl_test.go @@ -428,3 +428,112 @@ func Test_SchemaToSpannerSequenceHelper(t *testing.T) { assert.Equal(t, expectedConv, conv) } } + +func TestSpannerSchemaApplyExpressions(t *testing.T) { + makeConv := func() *internal.Conv { + conv := internal.MakeConv() + conv.SchemaIssues = make(map[string]internal.TableIssues) + conv.SchemaIssues["table1"] = internal.TableIssues{ + ColumnLevelIssues: make(map[string][]internal.SchemaIssue), + } + conv.SpSchema = ddl.Schema{ + "table1": { + ColDefs: map[string]ddl.ColumnDef{ + "col1": {}, + }, + }, + } + return conv + } + + makeResultConv := func(SpSchema ddl.Schema, SchemaIssues map[string]internal.TableIssues) *internal.Conv { + conv := internal.MakeConv() + conv.SpSchema = SpSchema + conv.SchemaIssues = SchemaIssues + return conv + } + + testCases := []struct { + name string + conv *internal.Conv + expressions internal.VerifyExpressionsOutput + expectedConv *internal.Conv + }{ + { + name: "successful default value application", + conv: makeConv(), + expressions: internal.VerifyExpressionsOutput{ + ExpressionVerificationOutputList: []internal.ExpressionVerificationOutput{ + { + Result: true, + ExpressionDetail: internal.ExpressionDetail{ + Type: "DEFAULT", + ExpressionId: "expr1", + Expression: "SELECT 1", + Metadata: map[string]string{"TableId": "table1", "ColId": "col1"}, + }, + }, + }, + }, + expectedConv: makeResultConv( + ddl.Schema{ + "table1": { + ColDefs: map[string]ddl.ColumnDef{ + "col1": { + DefaultValue: ddl.DefaultValue{ + IsPresent: true, + Value: ddl.Expression{ + ExpressionId: "expr1", + Statement: "SELECT 1", + }, + }, + }, + }, + }, + }, map[string]internal.TableIssues{ + "table1": { + ColumnLevelIssues: make(map[string][]internal.SchemaIssue), + }, + }), + }, + { + name: "failed default value application", + conv: makeConv(), + expressions: internal.VerifyExpressionsOutput{ + ExpressionVerificationOutputList: []internal.ExpressionVerificationOutput{ + { + Result: false, + ExpressionDetail: internal.ExpressionDetail{ + Type: "DEFAULT", + ExpressionId: "expr1", + Expression: "SELECT 1", + Metadata: map[string]string{"TableId": "table1", "ColId": "col1"}, + }, + }, + }, + }, + expectedConv: makeResultConv( + ddl.Schema{ + "table1": { + ColDefs: map[string]ddl.ColumnDef{ + "col1": {}, + }, + }, + }, + map[string]internal.TableIssues{ + "table1": { + ColumnLevelIssues: map[string][]internal.SchemaIssue{ + "col1": {internal.DefaultValue}, + }, + }, + }), + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + spannerSchemaApplyExpressions(tc.conv, tc.expressions) + assert.Equal(t, tc.expectedConv, tc.conv) + }) + } +} diff --git a/sources/common/utils.go b/sources/common/utils.go index c36c5b533..4581bd287 100644 --- a/sources/common/utils.go +++ b/sources/common/utils.go @@ -72,6 +72,21 @@ func GetSortedTableIdsBySrcName(srcSchema map[string]schema.Table) []string { return sortedTableIds } +func GetSortedTableIdsBySpName(spSchema ddl.Schema) []string { + tableNameIdMap := map[string]string{} + tableNames := []string{} + sortedTableIds := []string{} + for id, spTable := range spSchema { + tableNames = append(tableNames, spTable.Name) + tableNameIdMap[spTable.Name] = id + } + sort.Strings(tableNames) + for _, name := range tableNames { + sortedTableIds = append(sortedTableIds, tableNameIdMap[name]) + } + return sortedTableIds +} + func (uo *UtilsOrderImpl) initPrimaryKeyOrder(conv *internal.Conv) { for k, table := range conv.SrcSchema { for i := range table.PrimaryKeys { diff --git a/sources/common/utils_test.go b/sources/common/utils_test.go index 68fe6ad1c..b593c7c7a 100644 --- a/sources/common/utils_test.go +++ b/sources/common/utils_test.go @@ -270,3 +270,33 @@ func TestPrepareValues(t *testing.T) { assert.Equal(t, tc.expectedValues, res) } } + +func TestGetSortedTableIdsBySpName(t *testing.T) { + testCases := []struct { + name string + spSchema ddl.Schema + expectedIds []string + }{ + { + name: "multiple tables", + spSchema: ddl.Schema{ + "table2": {Name: "TableB"}, + "table1": {Name: "TableA"}, + "table3": {Name: "TableC"}, + }, + expectedIds: []string{"table1", "table2", "table3"}, + }, + { + name: "no tables", + spSchema: ddl.Schema{}, + expectedIds: []string{}, + }, + } + + for _, tc := range testCases { + t.Run(tc.name, func(t *testing.T) { + sortedIds := GetSortedTableIdsBySpName(tc.spSchema) + assert.Equal(t, tc.expectedIds, sortedIds) + }) + } +} diff --git a/sources/mysql/infoschema.go b/sources/mysql/infoschema.go index 87458188f..0cb4ad7c8 100644 --- a/sources/mysql/infoschema.go +++ b/sources/mysql/infoschema.go @@ -219,13 +219,26 @@ func (isi InfoSchemaImpl) GetColumns(conv *internal.Conv, table common.SchemaAnd } else { colAutoGen = ddl.AutoGenCol{} } + + defaultVal := ddl.DefaultValue{ + IsPresent: colDefault.Valid, + Value: ddl.Expression{}, + } + if colDefault.Valid { + defaultVal.Value = ddl.Expression{ + ExpressionId: internal.GenerateExpressionId(), + Statement: common.SanitizeDefaultValue(colDefault.String, dataType, colExtra.String == constants.DEFAULT_GENERATED), + } + } + c := schema.Column{ - Id: colId, - Name: colName, - Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale), - NotNull: common.ToNotNull(conv, isNullable), - Ignored: ignored, - AutoGen: colAutoGen, + Id: colId, + Name: colName, + Type: toType(dataType, columnType, charMaxLen, numericPrecision, numericScale), + NotNull: common.ToNotNull(conv, isNullable), + Ignored: ignored, + AutoGen: colAutoGen, + DefaultValue: defaultVal, } colDefs[colId] = c colIds = append(colIds, colId) diff --git a/sources/mysql/infoschema_test.go b/sources/mysql/infoschema_test.go index a61c04580..37836f382 100644 --- a/sources/mysql/infoschema_test.go +++ b/sources/mysql/infoschema_test.go @@ -69,8 +69,8 @@ func TestProcessSchemaMYSQL(t *testing.T) { args: []driver.Value{"test", "user"}, cols: []string{"column_name", "data_type", "column_type", "is_nullable", "column_default", "character_maximum_length", "numeric_precision", "numeric_scale", "extra"}, rows: [][]driver.Value{ - {"user_id", "text", "text", "NO", nil, nil, nil, nil, nil}, - {"name", "text", "text", "NO", nil, nil, nil, nil, nil}, + {"user_id", "text", "text", "NO", "uuid()", nil, nil, nil, constants.DEFAULT_GENERATED}, + {"name", "text", "text", "NO", "default_name", nil, nil, nil, nil}, {"ref", "bigint", "bigint", "NO", nil, nil, nil, nil, nil}}, }, // db call to fetch index happens after fetching of column @@ -233,7 +233,7 @@ func TestProcessSchemaMYSQL(t *testing.T) { "test": schema.Table{Name: "test", Schema: "test", ColIds: []string{"id", "s", "txt", "b", "bs", "bl", "c", "c8", "d", "dec", "f8", "f4", "i8", "i4", "i2", "si", "ts", "tz", "vc", "vc6"}, ColDefs: map[string]schema.Column{ "b": schema.Column{Name: "b", Type: schema.Type{Name: "boolean", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "bl": schema.Column{Name: "bl", Type: schema.Type{Name: "blob", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, - "bs": schema.Column{Name: "bs", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, + "bs": schema.Column{Name: "bs", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{IsPresent: true, Value: ddl.Expression{ExpressionId: "e27", Statement: "nextval('test11_bs_seq'::regclass)"}}}, "c": schema.Column{Name: "c", Type: schema.Type{Name: "char", Mods: []int64{1}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "c8": schema.Column{Name: "c8", Type: schema.Type{Name: "char", Mods: []int64{8}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "d": schema.Column{Name: "d", Type: schema.Type{Name: "date", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, @@ -241,11 +241,11 @@ func TestProcessSchemaMYSQL(t *testing.T) { "f4": schema.Column{Name: "f4", Type: schema.Type{Name: "float", Mods: []int64{24}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "f8": schema.Column{Name: "f8", Type: schema.Type{Name: "double", Mods: []int64{53}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "i2": schema.Column{Name: "i2", Type: schema.Type{Name: "smallint", Mods: []int64{16}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, - "i4": schema.Column{Name: "i4", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", AutoGen: ddl.AutoGenCol{Name: "Sequence34", GenerationType: constants.AUTO_INCREMENT}}, + "i4": schema.Column{Name: "i4", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", AutoGen: ddl.AutoGenCol{Name: "Sequence37", GenerationType: constants.AUTO_INCREMENT}}, "i8": schema.Column{Name: "i8", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "id": schema.Column{Name: "id", Type: schema.Type{Name: "bigint", Mods: []int64{64}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "s": schema.Column{Name: "s", Type: schema.Type{Name: "set", Mods: []int64(nil), ArrayBounds: []int64{-1}}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, - "si": schema.Column{Name: "si", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, + "si": schema.Column{Name: "si", Type: schema.Type{Name: "integer", Mods: []int64{32}, ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{IsPresent: true, Value: ddl.Expression{ExpressionId: "e40", Statement: "nextval('test11_s_seq'::regclass)"}}}, "ts": schema.Column{Name: "ts", Type: schema.Type{Name: "datetime", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "txt": schema.Column{Name: "txt", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, "tz": schema.Column{Name: "tz", Type: schema.Type{Name: "timestamp", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: false, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, @@ -262,9 +262,9 @@ func TestProcessSchemaMYSQL(t *testing.T) { ForeignKeys: []schema.ForeignKey(nil), Indexes: []schema.Index(nil), Id: ""}, "user": schema.Table{Name: "user", Schema: "test", ColIds: []string{"user_id", "name", "ref"}, ColDefs: map[string]schema.Column{ - "name": schema.Column{Name: "name", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, + "name": schema.Column{Name: "name", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{Value: ddl.Expression{ExpressionId: "e6", Statement: "'default_name'"}, IsPresent: true}}, "ref": schema.Column{Name: "ref", Type: schema.Type{Name: "bigint", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}, - "user_id": schema.Column{Name: "user_id", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: false, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: ""}}, + "user_id": schema.Column{Name: "user_id", Type: schema.Type{Name: "text", Mods: []int64(nil), ArrayBounds: []int64(nil)}, NotNull: true, Ignored: schema.Ignored{Check: false, Identity: false, Default: true, Exclusion: false, ForeignKey: false, AutoIncrement: false}, Id: "", DefaultValue: ddl.DefaultValue{Value: ddl.Expression{ExpressionId: "e4", Statement: "uuid()"}, IsPresent: true}}}, PrimaryKeys: []schema.Key{schema.Key{ColId: "user_id", Desc: false, Order: 0}}, ForeignKeys: []schema.ForeignKey{schema.ForeignKey{Name: "fk_test", ColIds: []string{"ref"}, ReferTableId: "test", ReferColumnIds: []string{"id"}, OnUpdate: constants.FK_CASCADE, OnDelete: constants.FK_SET_NULL, Id: ""}}, Indexes: []schema.Index(nil), Id: ""}} @@ -395,7 +395,7 @@ func TestProcessData_MultiCol(t *testing.T) { } internal.AssertSpSchema(conv, t, expectedSchema, stripSchemaComments(conv.SpSchema)) columnLevelIssues := map[string][]internal.SchemaIssue{ - "c49": []internal.SchemaIssue{ + "c53": []internal.SchemaIssue{ 2, }, } diff --git a/spanner/ddl/ast.go b/spanner/ddl/ast.go index 9490a89e3..91646281a 100644 --- a/spanner/ddl/ast.go +++ b/spanner/ddl/ast.go @@ -180,12 +180,13 @@ func (ty Type) PGPrintColumnDefType() string { // column_def: // column_name type [NOT NULL] [options_def] type ColumnDef struct { - Name string - T Type - NotNull bool - Comment string - Id string - AutoGen AutoGenCol + Name string + T Type + NotNull bool + Comment string + Id string + AutoGen AutoGenCol + DefaultValue DefaultValue } // Config controls how AST nodes are printed (aka unparsed). @@ -410,6 +411,45 @@ type AutoGenCol struct { GenerationType string } +// DefaultValue represents a Default value. +type DefaultValue struct { + IsPresent bool + Value Expression +} + +type Expression struct { + ExpressionId string + Statement string +} + +func (dv DefaultValue) PrintDefaultValue(ty Type) string { + if !dv.IsPresent { + return "" + } + var value string + switch ty.Name { + case "FLOAT32", "NUMERIC", "BOOL": + value = fmt.Sprintf(" DEFAULT (CAST(%s AS %s))", dv.Value.Statement, ty.Name) + default: + value = " DEFAULT (" + dv.Value.Statement + ")" + } + return value +} + +func (dv DefaultValue) PGPrintDefaultValue(ty Type) string { + if !dv.IsPresent { + return "" + } + var value string + switch ty.Name { + case "FLOAT8", "FLOAT4", "REAL", "NUMERIC", "DECIMAL", "BOOL": + value = fmt.Sprintf(" DEFAULT (CAST(%s AS %s))", dv.Value.Statement, ty.Name) + default: + value = " DEFAULT (" + dv.Value.Statement + ")" + } + return value +} + func (agc AutoGenCol) PrintAutoGenCol() string { if agc.Name == constants.UUID && agc.GenerationType == "Pre-defined" { return " DEFAULT (GENERATE_UUID())" diff --git a/spanner/ddl/ast_test.go b/spanner/ddl/ast_test.go index d6e53f96b..b9e6a510e 100644 --- a/spanner/ddl/ast_test.go +++ b/spanner/ddl/ast_test.go @@ -496,6 +496,96 @@ func TestPrintForeignKeyAlterTable(t *testing.T) { } } +func TestPrintDefaultValue(t *testing.T) { + tests := []struct { + name string + dv DefaultValue + ty Type + expected string + }{ + { + name: "default value present", + dv: DefaultValue{ + IsPresent: true, + Value: Expression{Statement: "(`col1` + 1)"}, + }, + ty: Type{ + Name: "INT64", + }, + expected: " DEFAULT ((`col1` + 1))", + }, + { + name: "default value present", + dv: DefaultValue{ + IsPresent: true, + Value: Expression{Statement: "(`col1` + 1)"}, + }, + ty: Type{ + Name: "NUMERIC", + }, + expected: " DEFAULT (CAST((`col1` + 1) AS NUMERIC))", + }, + { + name: "empty default value", + dv: DefaultValue{}, + ty: Type{ + Name: "INT64", + }, + expected: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, tc.dv.PrintDefaultValue(tc.ty)) + }) + } +} + +func TestPGPrintDefaultValue(t *testing.T) { + tests := []struct { + name string + dv DefaultValue + ty Type + expected string + }{ + { + name: "default value present", + dv: DefaultValue{ + IsPresent: true, + Value: Expression{Statement: "(`col1` + 1)"}, + }, + ty: Type{ + Name: "INT64", + }, + expected: " DEFAULT ((`col1` + 1))", + }, + { + name: "default value present", + dv: DefaultValue{ + IsPresent: true, + Value: Expression{Statement: "(`col1` + 1)"}, + }, + ty: Type{ + Name: "NUMERIC", + }, + expected: " DEFAULT (CAST((`col1` + 1) AS NUMERIC))", + }, + { + name: "empty default value", + dv: DefaultValue{}, + ty: Type{ + Name: "INT64", + }, + expected: "", + }, + } + for _, tc := range tests { + t.Run(tc.name, func(t *testing.T) { + assert.Equal(t, tc.expected, tc.dv.PGPrintDefaultValue(tc.ty)) + }) + } +} + func TestPrintAutoGenCol(t *testing.T) { tests := []struct { agc AutoGenCol diff --git a/test_data/mysql_shard_streaming.cfg b/test_data/mysql_shard_streaming.cfg index 7332a2828..a424914ec 100644 --- a/test_data/mysql_shard_streaming.cfg +++ b/test_data/mysql_shard_streaming.cfg @@ -11,7 +11,7 @@ "dataShards":[ { "dataShardId":"physical1", - "tmpDir":"gs://smt-test/", + "tmpDir":"gs://smt-test-bucket/", "streamLocation":"asia-east2", "srcConnectionProfile":{ "name":"test1", @@ -48,4 +48,4 @@ } ] } -} \ No newline at end of file +} diff --git a/ui/dist/ui/index.html b/ui/dist/ui/index.html index cc74a37a6..952400454 100644 --- a/ui/dist/ui/index.html +++ b/ui/dist/ui/index.html @@ -7,10 +7,10 @@ - - + +