From 93cc109a57a48e63d886cd53963fc59c837d422a Mon Sep 17 00:00:00 2001 From: Kevin Joiner <10265309+KevinJoiner@users.noreply.github.com> Date: Wed, 24 Apr 2024 17:01:10 -0400 Subject: [PATCH] Update migrations. * Remove old migrations. * Make goose table replicated. * Allow custom migration names. * Update README.md --- Makefile | 2 +- README.md | 17 ++-- cmd/codegen/codegen.go | 7 ++ internal/codegen/migration/migration.go | 43 +++++++--- internal/codegen/migration/migration.tmpl | 6 +- pkg/migrations/20240402091429_init.go | 79 ------------------- .../20240405125845_dynamic_signals.go | 45 ----------- pkg/migrations/20240411014301_drop_subject.go | 32 -------- .../20240411014401_remove_value_array.go | 56 ------------- .../20240416200430_drop_dimo_migration.go | 44 ----------- ...0240419162127_drop_valueArray_migration.go | 44 ----------- ...40424204905_init_goose_table_migration.go} | 28 ++++--- ... 20240424204906_signal_table_migration.go} | 22 +++--- pkg/migrations/migrations.go | 5 +- pkg/runner/runner.go | 3 +- 15 files changed, 85 insertions(+), 348 deletions(-) delete mode 100644 pkg/migrations/20240402091429_init.go delete mode 100644 pkg/migrations/20240405125845_dynamic_signals.go delete mode 100644 pkg/migrations/20240411014301_drop_subject.go delete mode 100644 pkg/migrations/20240411014401_remove_value_array.go delete mode 100644 pkg/migrations/20240416200430_drop_dimo_migration.go delete mode 100644 pkg/migrations/20240419162127_drop_valueArray_migration.go rename pkg/migrations/{20240411200712_clustered_migration.go => 20240424204905_init_goose_table_migration.go} (51%) rename pkg/migrations/{20240423004039_local_updates_migration.go => 20240424204906_signal_table_migration.go} (64%) diff --git a/Makefile b/Makefile index 25ffbb4..1872667 100644 --- a/Makefile +++ b/Makefile @@ -46,7 +46,7 @@ format: @golangci-lint run --fix migration: build - ./bin/codegen -output=./pkg/migrations -package=migrations -generators=migration + ./bin/codegen -output=./pkg/migrations -package=migrations -generators=migration -migration.file-name="${name}" tools-golangci-lint: @mkdir -p bin diff --git a/README.md b/README.md index f0081a6..c95f1e1 100644 --- a/README.md +++ b/README.md @@ -8,12 +8,19 @@ Welcome to the **Model Garage**, a Golang toolkit for managing and working with ## Features -1. **Model Creation**: Create models from vspec CSV schemas, represented as Go structs and matching Clickhouse tables. - +1. **Model Creation**: Create models from vspec CSV schemas, represented as Go structs. 2. **JSON Conversion**: Easily convert JSON data for your models with automatically generated and customizable conversion functions. -3. **Random Data Generation**: [**Coming Soon**] Quickly generate models populated with random data for testing or sample data. +## Migrations + +To create a new migration, run the following command: +```bash +Make migration name= +``` + +This will create a new migration file the given name in the `migrations` directory. +this creation should be used over the goose binary to ensure expected behavior of embedded migrations. ## Repo structure @@ -32,7 +39,7 @@ go run github.com/DIMO-Network/model-garage/cmd/codegen -output=pkg/vss -genera The Model generation is handled by packages in `internal/codegen`. They are responsible for creating Go structs, Clickhouse tables, and conversion functions from the vspec CSV schema and definitions file. definitions file is a YAML file that specifies the conversions for each field in the vspec schema. The conversion functions are meant to be overridden with custom logic as needed. When generation is re-run, the conversion functions are not overwritten. Below is an example of a definitions file: ```yaml - # vspecName: The name of the VSpec field in the VSS schema +# vspecName: The name of the VSpec field in the VSS schema - vspecName: DIMO.DefinitionID # isArray: Whether the field is an array or not @@ -67,7 +74,6 @@ The Model generation is handled by packages in `internal/codegen`. They are resp - VehicleNonLocationData ``` - ##### Generation Process 1. First, the vspec CSV schema and definitions file are parsed. @@ -77,4 +83,3 @@ The Model generation is handled by packages in `internal/codegen`. They are resp **Conversion Functions** For each field, a conversion function is created. If a conversion is specified in the definitions file, the conversion function will use the specified conversion. If no conversion is specified, the conversion info function will assume a direct copy. The conversion functions are meant to be overridden with custom logic as needed. When generation is re-run, the conversion functions are not overwritten. - diff --git a/cmd/codegen/codegen.go b/cmd/codegen/codegen.go index fb0774d..3781e3d 100644 --- a/cmd/codegen/codegen.go +++ b/cmd/codegen/codegen.go @@ -12,6 +12,7 @@ import ( "github.com/DIMO-Network/model-garage/internal/codegen/convert" "github.com/DIMO-Network/model-garage/internal/codegen/graphql" + "github.com/DIMO-Network/model-garage/internal/codegen/migration" "github.com/DIMO-Network/model-garage/pkg/runner" "github.com/DIMO-Network/model-garage/schema" ) @@ -28,6 +29,9 @@ func main() { // GQL flags gqlOutFile := flag.String("graphql.output-file", "", "Path of the generate gql file that is appened to the outputDir.") gqlTemplateFile := flag.String("graphql.template-file", "", "Path to the template file. Which is executed with codegen.TemplateData data.") + // Migration flags + migrationFileName := flag.String("migration.file-name", "", "Name of the migration file. Default is the model name.") + flag.Parse() var vspecReader io.Reader @@ -67,6 +71,9 @@ func main() { Convert: convert.Config{ WithTest: *withTest, }, + Migration: migration.Config{ + FileName: *migrationFileName, + }, } err := runner.Execute(vspecReader, definitionReader, gens, cfg) diff --git a/internal/codegen/migration/migration.go b/internal/codegen/migration/migration.go index 35497fb..77e27f5 100644 --- a/internal/codegen/migration/migration.go +++ b/internal/codegen/migration/migration.go @@ -11,19 +11,38 @@ import ( "time" "github.com/DIMO-Network/model-garage/internal/codegen" + "golang.org/x/text/cases" + "golang.org/x/text/language" ) -var migrationFileFormat = "%s_%s_migration.go" +var ( + delemReplacer = strings.NewReplacer("_", " ", "-", " ", ".", " ") + titleCaser = cases.Title(language.AmericanEnglish, cases.NoLower) + lowerCaser = cases.Lower(language.AmericanEnglish) +) -const timestampFormat = "20060102150405" +const ( + migrationFileFormat = "%s_%s_migration.go" + timestampFormat = "20060102150405" +) //go:embed migration.tmpl var migrationFileTemplate string +// Config is the configuration for the migration generator. +type Config struct { + // fileName is the name of the migration file. + FileName string +} + // Generate creates a new ClickHouse table file. -func Generate(tmplData *codegen.TemplateData, outputDir string) error { +func Generate(tmplData *codegen.TemplateData, outputDir string, cfg Config) error { version := time.Now().UTC().Format(timestampFormat) - migrationTempl, err := createMigrationTemplate(version) + fileName := cfg.FileName + if fileName == "" { + fileName = tmplData.ModelName + } + migrationTempl, err := createMigrationTemplate(fileName) if err != nil { return err } @@ -33,7 +52,9 @@ func Generate(tmplData *codegen.TemplateData, outputDir string) error { if err != nil { return fmt.Errorf("error executing ClickHouse table template: %w", err) } - migrationFilePath := getFilePath(strings.ToLower(tmplData.ModelName), outputDir, version) + + fileName = delemReplacer.Replace(fileName) + migrationFilePath := getFilePath(fileName, outputDir, version) err = codegen.FormatAndWriteToFile(outBuf.Bytes(), migrationFilePath) if err != nil { return fmt.Errorf("error writing file: %w", err) @@ -42,11 +63,10 @@ func Generate(tmplData *codegen.TemplateData, outputDir string) error { return nil } -func createMigrationTemplate(version string) (*template.Template, error) { +func createMigrationTemplate(fileName string) (*template.Template, error) { + funcName := strings.ReplaceAll(titleCaser.String(fileName), " ", "") tmpl, err := template.New("migrationTemplate").Funcs(template.FuncMap{ - "escapeDesc": func(desc string) string { return strings.ReplaceAll(desc, `'`, `\'`) }, - "lower": strings.ToLower, - "version": func() string { return version }, + "funcName": func() string { return funcName }, }).Parse(migrationFileTemplate) if err != nil { return nil, fmt.Errorf("error parsing ClickHouse table template: %w", err) @@ -54,7 +74,8 @@ func createMigrationTemplate(version string) (*template.Template, error) { return tmpl, nil } -func getFilePath(modelName, outputDir string, version string) string { - migrationFileName := fmt.Sprintf(migrationFileFormat, version, modelName) +func getFilePath(fileName, outputDir string, version string) string { + noSpaceName := lowerCaser.String(strings.ReplaceAll(fileName, " ", "_")) + migrationFileName := fmt.Sprintf(migrationFileFormat, version, noSpaceName) return filepath.Clean(filepath.Join(outputDir, migrationFileName)) } diff --git a/internal/codegen/migration/migration.tmpl b/internal/codegen/migration/migration.tmpl index 7351bdd..8b35041 100644 --- a/internal/codegen/migration/migration.tmpl +++ b/internal/codegen/migration/migration.tmpl @@ -9,12 +9,12 @@ import ( func init() { _, filename, _, _ := runtime.Caller(0) - registerFunc := func() { goose.AddNamedMigrationContext(filename, upCommand{{ version }}, downCommand{{ version }}) } + registerFunc := func() { goose.AddNamedMigrationContext(filename, up{{ funcName }}, down{{ funcName }}) } registerFuncs = append(registerFuncs, registerFunc) registerFunc() } -func upCommand{{ version }}(ctx context.Context, tx *sql.Tx) error { +func up{{ funcName }}(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is applied. upStatements := []string{ } @@ -27,7 +27,7 @@ func upCommand{{ version }}(ctx context.Context, tx *sql.Tx) error { return nil } -func downCommand{{ version }}(ctx context.Context, tx *sql.Tx) error { +func down{{ funcName }}(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is rolled back. downStatements :=[]string{ } diff --git a/pkg/migrations/20240402091429_init.go b/pkg/migrations/20240402091429_init.go deleted file mode 100644 index 146bb90..0000000 --- a/pkg/migrations/20240402091429_init.go +++ /dev/null @@ -1,79 +0,0 @@ -// Package migrations provides the database migration for the DIMO clickhouse database. -package migrations - -import ( - "context" - "database/sql" - "fmt" - "runtime" - - "github.com/pressly/goose/v3" -) - -func init() { - _, filename, _, _ := runtime.Caller(0) - addFunc := func() { goose.AddNamedMigrationContext(filename, upInit, downInit) } - registerFuncs = append(registerFuncs, addFunc) - addFunc() -} - -func upInit(ctx context.Context, tx *sql.Tx) error { - // This code is executed when the migration is applied. - _, err := tx.ExecContext(ctx, createDIMOtableStatement) - if err != nil { - return fmt.Errorf("failed to create dimo table: %w", err) - } - return nil -} - -func downInit(ctx context.Context, tx *sql.Tx) error { - // This code is executed when the migration is rolled back. - _, err := tx.ExecContext(ctx, "DROP TABLE IF EXISTS dimo") - if err != nil { - return fmt.Errorf("failed to drop dimo table: %w", err) - } - return nil -} - -//nolint:misspell // false positive -const createDIMOtableStatement = `CREATE TABLE IF NOT EXISTS dimo ( - DefinitionID String COMMENT 'ID for the vehicles definition', - Source String COMMENT 'where the data was sourced from', - Subject String COMMENT 'subjet of this vehicle data', - Timestamp DateTime('UTC') COMMENT 'timestamp of when this data was colllected', - Type String COMMENT 'type of data collected', - Vehicle_Chassis_Axle_Row1_Wheel_Left_Tire_Pressure UInt16 COMMENT 'Tire pressure in kilo-Pascal.', - Vehicle_Chassis_Axle_Row1_Wheel_Right_Tire_Pressure UInt16 COMMENT 'Tire pressure in kilo-Pascal.', - Vehicle_Chassis_Axle_Row2_Wheel_Left_Tire_Pressure UInt16 COMMENT 'Tire pressure in kilo-Pascal.', - Vehicle_Chassis_Axle_Row2_Wheel_Right_Tire_Pressure UInt16 COMMENT 'Tire pressure in kilo-Pascal.', - Vehicle_CurrentLocation_Altitude Float64 COMMENT 'Current altitude relative to WGS 84 reference ellipsoid, as measured at the position of GNSS receiver antenna.', - Vehicle_CurrentLocation_Latitude Float64 COMMENT 'Current latitude of vehicle in WGS 84 geodetic coordinates, as measured at the position of GNSS receiver antenna.', - Vehicle_CurrentLocation_Longitude Float64 COMMENT 'Current longitude of vehicle in WGS 84 geodetic coordinates, as measured at the position of GNSS receiver antenna.', - Vehicle_CurrentLocation_Timestamp DateTime('UTC') COMMENT 'Timestamp from GNSS system for current location, formatted according to ISO 8601 with UTC time zone.', - Vehicle_Exterior_AirTemperature Float32 COMMENT 'Air temperature outside the vehicle.', - Vehicle_LowVoltageBattery_CurrentVoltage Float32 COMMENT 'Current Voltage of the low voltage battery.', - Vehicle_OBD_BarometricPressure Float32 COMMENT 'PID 33 - Barometric pressure', - Vehicle_OBD_EngineLoad Float32 COMMENT 'PID 04 - Engine load in percent - 0 = no load, 100 = full load', - Vehicle_OBD_IntakeTemp Float32 COMMENT 'PID 0F - Intake temperature', - Vehicle_OBD_RunTime Float32 COMMENT 'PID 1F - Engine run time', - Vehicle_Powertrain_CombustionEngine_ECT Int16 COMMENT 'Engine coolant temperature.', - Vehicle_Powertrain_CombustionEngine_EngineOilLevel String COMMENT 'Engine oil level.', - Vehicle_Powertrain_CombustionEngine_Speed UInt16 COMMENT 'Engine speed measured as rotations per minute.', - Vehicle_Powertrain_CombustionEngine_TPS UInt8 COMMENT 'Current throttle position.', - Vehicle_Powertrain_FuelSystem_AbsoluteLevel Float32 COMMENT 'Current available fuel in the fuel tank expressed in liters.', - Vehicle_Powertrain_FuelSystem_SupportedFuelTypes Array(String) COMMENT 'High level information of fuel types supported', - Vehicle_Powertrain_Range UInt32 COMMENT 'Remaining range in meters using all energy sources available in the vehicle.', - Vehicle_Powertrain_TractionBattery_Charging_ChargeLimit UInt8 COMMENT 'Target charge limit (state of charge) for battery.', - Vehicle_Powertrain_TractionBattery_Charging_IsCharging Bool COMMENT 'True if charging is ongoing. Charging is considered to be ongoing if energy is flowing from charger to vehicle.', - Vehicle_Powertrain_TractionBattery_GrossCapacity UInt16 COMMENT 'Gross capacity of the battery.', - Vehicle_Powertrain_TractionBattery_StateOfCharge_Current Float32 COMMENT 'Physical state of charge of the high voltage battery, relative to net capacity. This is not necessarily the state of charge being displayed to the customer.', - Vehicle_Powertrain_Transmission_TravelledDistance Float32 COMMENT 'Odometer reading, total distance travelled during the lifetime of the transmission.', - Vehicle_Powertrain_Type String COMMENT 'Defines the powertrain type of the vehicle.', - Vehicle_Speed Float32 COMMENT 'Vehicle speed.', - Vehicle_VehicleIdentification_Brand String COMMENT 'Vehicle brand or manufacturer.', - Vehicle_VehicleIdentification_Model String COMMENT 'Vehicle model.', - Vehicle_VehicleIdentification_Year UInt16 COMMENT 'Model year of the vehicle.', - VehicleID String COMMENT 'unque DIMO ID for the vehicle', -) -ENGINE = MergeTree() -ORDER BY (Subject, Timestamp)` diff --git a/pkg/migrations/20240405125845_dynamic_signals.go b/pkg/migrations/20240405125845_dynamic_signals.go deleted file mode 100644 index 31838e1..0000000 --- a/pkg/migrations/20240405125845_dynamic_signals.go +++ /dev/null @@ -1,45 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "fmt" - "runtime" - - "github.com/pressly/goose/v3" -) - -func init() { - _, filename, _, _ := runtime.Caller(0) - addFunc := func() { goose.AddNamedMigrationContext(filename, upDynamicSignals, downDynamicSignals) } - registerFuncs = append(registerFuncs, addFunc) - addFunc() -} - -func upDynamicSignals(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, createVehicleSigsTable) - if err != nil { - return fmt.Errorf("failed to create dimo table: %w", err) - } - return nil -} - -func downDynamicSignals(ctx context.Context, tx *sql.Tx) error { - dropStmt := "DROP TABLE IF EXISTS signal" - _, err := tx.ExecContext(ctx, dropStmt) - if err != nil { - return fmt.Errorf("failed to drop signal table: %w", err) - } - return nil -} - -const createVehicleSigsTable = `CREATE TABLE IF NOT EXISTS signal( - TokenID UInt32 COMMENT 'tokenID of this device data.', - Subject String COMMENT 'subjet of this vehicle data.', - Timestamp DateTime('UTC') COMMENT 'timestamp of when this data was colllected.', - Name LowCardinality(String) COMMENT 'name of the signal collected.', - ValueString Array(String) COMMENT 'value of the signal collected.', - ValueNumber Array(Float64) COMMENT 'value of the signal collected.', -) -ENGINE = MergeTree() -ORDER BY (TokenID, Timestamp, Name)` diff --git a/pkg/migrations/20240411014301_drop_subject.go b/pkg/migrations/20240411014301_drop_subject.go deleted file mode 100644 index 281e5da..0000000 --- a/pkg/migrations/20240411014301_drop_subject.go +++ /dev/null @@ -1,32 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "runtime" - - "github.com/pressly/goose/v3" -) - -func init() { - _, filename, _, _ := runtime.Caller(0) - addFunc := func() { goose.AddNamedMigrationContext(filename, upRemoveDropSubject, downRemoveDropSubject) } - registerFuncs = append(registerFuncs, addFunc) - addFunc() -} - -func upRemoveDropSubject(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "ALTER TABLE signal DROP COLUMN Subject") - if err != nil { - return err - } - return nil -} - -func downRemoveDropSubject(ctx context.Context, tx *sql.Tx) error { - _, err := tx.ExecContext(ctx, "ALTER TABLE signal ADD COLUMN Subject String After TokenID") - if err != nil { - return err - } - return nil -} diff --git a/pkg/migrations/20240411014401_remove_value_array.go b/pkg/migrations/20240411014401_remove_value_array.go deleted file mode 100644 index 015738b..0000000 --- a/pkg/migrations/20240411014401_remove_value_array.go +++ /dev/null @@ -1,56 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "runtime" - - "github.com/pressly/goose/v3" -) - -func init() { - _, filename, _, _ := runtime.Caller(0) - addFunc := func() { goose.AddNamedMigrationContext(filename, upRemoveValueArray, downRemoveValueArray) } - registerFuncs = append(registerFuncs, addFunc) - addFunc() -} - -func upRemoveValueArray(ctx context.Context, tx *sql.Tx) error { - stmts := []string{ - "ALTER TABLE signal RENAME COLUMN ValueNumber TO tmp", - "ALTER TABLE signal ADD COLUMN ValueNumber Float64 AFTER Name", - "ALTER TABLE signal COMMENT COLUMN ValueNumber 'float64 value of the signal collected.'", - "ALTER TABLE signal UPDATE ValueNumber = arrayElement(tmp, 1) WHERE notEmpty(tmp) = 1", - "ALTER TABLE signal DROP COLUMN tmp", - "ALTER TABLE signal RENAME COLUMN ValueString TO ValueStringArray", - "ALTER TABLE signal COMMENT COLUMN ValueStringArray 'string array value of the signal collected.'", - "ALTER TABLE signal ADD COLUMN ValueString String AFTER ValueNumber", - "ALTER TABLE signal COMMENT COLUMN ValueString 'string value of the signal collected.'", - "ALTER TABLE signal UPDATE ValueString = arrayElement(ValueStringArray, 1) WHERE length(ValueStringArray) = 1", - } - for _, stmt := range stmts { - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - } - return nil -} - -func downRemoveValueArray(ctx context.Context, tx *sql.Tx) error { - stmts := []string{ - "ALTER TABLE signal DROP COLUMN ValueString", - "ALTER TABLE signal RENAME COLUMN ValueStringArray TO ValueString", - "ALTER TABLE signal RENAME COLUMN ValueNumber TO tmp", - "ALTER TABLE signal ADD COLUMN ValueNumber Array(Float64)", - "ALTER TABLE signal UPDATE ValueNumber = [tmp] WHERE tmp != 0", - "ALTER TABLE signal DROP COLUMN tmp", - } - for _, stmt := range stmts { - _, err := tx.ExecContext(ctx, stmt) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/migrations/20240416200430_drop_dimo_migration.go b/pkg/migrations/20240416200430_drop_dimo_migration.go deleted file mode 100644 index bb369a1..0000000 --- a/pkg/migrations/20240416200430_drop_dimo_migration.go +++ /dev/null @@ -1,44 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "runtime" - - "github.com/pressly/goose/v3" -) - -func init() { - _, filename, _, _ := runtime.Caller(0) - registerFunc := func() { goose.AddNamedMigrationContext(filename, upCommand20240416200430, downCommand20240416200430) } - registerFuncs = append(registerFuncs, registerFunc) - registerFunc() -} - -func upCommand20240416200430(ctx context.Context, tx *sql.Tx) error { - // This code is executed when the migration is applied. - upStatements := []string{ - "DROP TABLE IF EXISTS dimo", - } - for _, upStatement := range upStatements { - _, err := tx.ExecContext(ctx, upStatement) - if err != nil { - return err - } - } - return nil -} - -func downCommand20240416200430(ctx context.Context, tx *sql.Tx) error { - // This code is executed when the migration is rolled back. - downStatements := []string{ - createDIMOtableStatement, - } - for _, downStatement := range downStatements { - _, err := tx.ExecContext(ctx, downStatement) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/migrations/20240419162127_drop_valueArray_migration.go b/pkg/migrations/20240419162127_drop_valueArray_migration.go deleted file mode 100644 index 19ec28e..0000000 --- a/pkg/migrations/20240419162127_drop_valueArray_migration.go +++ /dev/null @@ -1,44 +0,0 @@ -package migrations - -import ( - "context" - "database/sql" - "runtime" - - "github.com/pressly/goose/v3" -) - -func init() { - _, filename, _, _ := runtime.Caller(0) - registerFunc := func() { goose.AddNamedMigrationContext(filename, upCommand20240419162127, downCommand20240419162127) } - registerFuncs = append(registerFuncs, registerFunc) - registerFunc() -} - -func upCommand20240419162127(ctx context.Context, tx *sql.Tx) error { - // This code is executed when the migration is applied. - upStatements := []string{ - "ALTER TABLE signal DROP COLUMN ValueStringArray", - } - for _, upStatement := range upStatements { - _, err := tx.ExecContext(ctx, upStatement) - if err != nil { - return err - } - } - return nil -} - -func downCommand20240419162127(ctx context.Context, tx *sql.Tx) error { - // This code is executed when the migration is rolled back. - downStatements := []string{ - "ALTER TABLE signal ADD COLUMN ValueStringArray Array(String)", - } - for _, downStatement := range downStatements { - _, err := tx.ExecContext(ctx, downStatement) - if err != nil { - return err - } - } - return nil -} diff --git a/pkg/migrations/20240411200712_clustered_migration.go b/pkg/migrations/20240424204905_init_goose_table_migration.go similarity index 51% rename from pkg/migrations/20240411200712_clustered_migration.go rename to pkg/migrations/20240424204905_init_goose_table_migration.go index 6c122b1..50b2bb6 100644 --- a/pkg/migrations/20240411200712_clustered_migration.go +++ b/pkg/migrations/20240424204905_init_goose_table_migration.go @@ -10,19 +10,27 @@ import ( func init() { _, filename, _, _ := runtime.Caller(0) - registerFunc := func() { goose.AddNamedMigrationContext(filename, upCommand20240411200712, downCommand20240411200712) } + registerFunc := func() { goose.AddNamedMigrationContext(filename, upInitGooseTable, downInitGooseTable) } registerFuncs = append(registerFuncs, registerFunc) registerFunc() } -func upCommand20240411200712(ctx context.Context, tx *sql.Tx) error { +func upInitGooseTable(ctx context.Context, tx *sql.Tx) error { + const createStmt = ` + CREATE TABLE IF NOT EXISTS goose_db_version ON CLUSTER '{cluster}' as tmp + ENGINE = ReplicatedMergeTree('/clickhouse/tables/{cluster}/{database}/{table}/{uuid}', '{replica}') + ORDER BY date;` + // This code is executed when the migration is applied. upStatements := []string{ - "RENAME TABLE signal TO local_signal ON CLUSTER '{cluster}'", - clusterSignalCreateStmt, - distributedSignalCreateStmt, - "INSERT INTO signal select * FROM clusterAllReplicas('{cluster}', default.local_signal)", - "DROP TABLE local_signal ON CLUSTER '{cluster}'", + // move goose table to a local tmp table + "RENAME TABLE goose_db_version TO tmp", + // make goos table that is replicated across the cluster + createStmt, + // copy data from old goose table to new goose table + "INSERT INTO goose_db_version SELECT * FROM tmp", + // drop the old goose table + "DROP TABLE tmp", } for _, upStatement := range upStatements { _, err := tx.ExecContext(ctx, upStatement) @@ -33,7 +41,7 @@ func upCommand20240411200712(ctx context.Context, tx *sql.Tx) error { return nil } -func downCommand20240411200712(ctx context.Context, tx *sql.Tx) error { +func downInitGooseTable(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is rolled back. downStatements := []string{} for _, downStatement := range downStatements { @@ -44,7 +52,3 @@ func downCommand20240411200712(ctx context.Context, tx *sql.Tx) error { } return nil } - -var clusterSignalCreateStmt = `CREATE table signal_shard on CLUSTER '{cluster}' as local_signal -ENGINE = ReplicatedReplacingMergeTree('/clickhouse/tables/{shard}/{database}/{table}', '{replica}') -ORDER BY (TokenID, Timestamp, Name)` diff --git a/pkg/migrations/20240423004039_local_updates_migration.go b/pkg/migrations/20240424204906_signal_table_migration.go similarity index 64% rename from pkg/migrations/20240423004039_local_updates_migration.go rename to pkg/migrations/20240424204906_signal_table_migration.go index 0765411..22a51f9 100644 --- a/pkg/migrations/20240423004039_local_updates_migration.go +++ b/pkg/migrations/20240424204906_signal_table_migration.go @@ -3,7 +3,6 @@ package migrations import ( "context" "database/sql" - "fmt" "runtime" "github.com/pressly/goose/v3" @@ -11,34 +10,31 @@ import ( func init() { _, filename, _, _ := runtime.Caller(0) - registerFunc := func() { goose.AddNamedMigrationContext(filename, upCommand20240423004039, downCommand20240423004039) } + registerFunc := func() { goose.AddNamedMigrationContext(filename, upSignalTable, downSignalTable) } registerFuncs = append(registerFuncs, registerFunc) registerFunc() } -func upCommand20240423004039(ctx context.Context, tx *sql.Tx) error { +func upSignalTable(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is applied. upStatements := []string{ - "DROP TABLE default.signal ON CLUSTER '{cluster}'", - "RENAME TABLE default.signal_shard TO default.signal_shard_old ON CLUSTER '{cluster}'", createSignalShardStmt, - "INSERT INTO default.signal_shard SELECT TokenID, Timestamp, Name, ValueNumber, ValueString FROM default.signal_shard_old", distributedSignalCreateStmt, - "DROP TABLE default.signal_shard_old ON CLUSTER '{cluster}'", } for _, upStatement := range upStatements { _, err := tx.ExecContext(ctx, upStatement) if err != nil { - return fmt.Errorf("failed to execute statment %s: %w", upStatement, err) + return err } } return nil } -func downCommand20240423004039(ctx context.Context, tx *sql.Tx) error { +func downSignalTable(ctx context.Context, tx *sql.Tx) error { // This code is executed when the migration is rolled back. downStatements := []string{ - "ALTER TABLE signal MODIFY COLUMN Timestamp Datetime('UTC')", + "DROP TABLE signal ON CLUSTER '{cluster}'", + "DROP TABLE signal_shard ON CLUSTER '{cluster}'", } for _, downStatement := range downStatements { _, err := tx.ExecContext(ctx, downStatement) @@ -49,9 +45,9 @@ func downCommand20240423004039(ctx context.Context, tx *sql.Tx) error { return nil } -var ( +const ( createSignalShardStmt = ` -CREATE TABLE default.signal_shard ON CLUSTER '{cluster}' +CREATE TABLE IF NOT EXISTS signal_shard ON CLUSTER '{cluster}' ( TokenID UInt32 COMMENT 'tokenID of this device data.', Timestamp DateTime64(6, 'UTC') COMMENT 'timestamp of when this data was colllected.', @@ -64,7 +60,7 @@ ORDER BY (TokenID, Timestamp, Name) ` distributedSignalCreateStmt = ` -CREATE TABLE signal ON CLUSTER '{cluster}' AS signal_shard +CREATE TABLE IF NOT EXISTS signal ON CLUSTER '{cluster}' AS signal_shard ENGINE = Distributed('{cluster}', default, signal_shard, TokenID) ` ) diff --git a/pkg/migrations/migrations.go b/pkg/migrations/migrations.go index 50e4a8b..a2c58de 100644 --- a/pkg/migrations/migrations.go +++ b/pkg/migrations/migrations.go @@ -49,7 +49,10 @@ func RunGoose(ctx context.Context, gooseArgs []string, db *sql.DB) error { if len(gooseArgs) > 1 { args = os.Args[1:] } - + // err := initMigrations(ctx, db) + // if err != nil { + // return fmt.Errorf("failed to init migrations for clickhouse cluster: %w", err) + // } SetMigrations() if err := goose.SetDialect("clickhouse"); err != nil { return fmt.Errorf("failed to set dialect: %w", err) diff --git a/pkg/runner/runner.go b/pkg/runner/runner.go index 1bd0fa9..6ccf16b 100644 --- a/pkg/runner/runner.go +++ b/pkg/runner/runner.go @@ -32,6 +32,7 @@ type Config struct { OutputDir string GraphQL graphql.Config Convert convert.Config + Migration migration.Config } // Execute runs the code generation tool. @@ -86,7 +87,7 @@ func Execute(vspecReader, definitionsReader io.Reader, generators []string, cfg } if slices.Contains(generators, AllGenerator) || slices.Contains(generators, MigrationGenerator) { - err = migration.Generate(tmplData, cfg.OutputDir) + err = migration.Generate(tmplData, cfg.OutputDir, cfg.Migration) if err != nil { return fmt.Errorf("failed to generate migration file: %w", err) }