Merge branch 'master' into clickhouse_create_database_migrations

.github/workflows/ci.yaml

@@ -9,10 +9,10 @@ jobs:
     name: lint
     runs-on: ubuntu-latest
     steps:
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: 1.19
-      - uses: actions/checkout@v3
+          go-version: "1.21.x"
+      - uses: actions/checkout@v4
       - name: golangci-lint
         uses: golangci/golangci-lint-action@v3
         with:
@@ -22,11 +22,11 @@ jobs:
     runs-on: ubuntu-latest
     strategy:
       matrix:
-        go: ["1.18.x", "1.19.x"]
+        go: ["1.20.x", "1.21.x"]
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
 
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
           go-version: ${{ matrix.go }}
 
@@ -60,27 +60,27 @@ jobs:
     # 3. When the workflow is triggered by a tag with `v` prefix
     if: ${{ success() && github.repository == 'golang-migrate/migrate' && startsWith(github.ref, 'refs/tags/v') }}
     steps:
-      - uses: actions/checkout@v3
+      - uses: actions/checkout@v4
         with:
           fetch-depth: 0
       - uses: ruby/setup-ruby@v1
         with:
           ruby-version: 2.7
-      - uses: actions/setup-go@v3
+      - uses: actions/setup-go@v5
         with:
-          go-version: "1.19.x"
+          go-version: "1.21.x"
 
-      - uses: docker/setup-qemu-action@v1
-      - uses: docker/setup-buildx-action@v1
-      - uses: docker/login-action@v1
+      - uses: docker/setup-qemu-action@v3
+      - uses: docker/setup-buildx-action@v3
+      - uses: docker/login-action@v3
         with:
           username: golangmigrate
           password: ${{ secrets.DOCKERHUB_TOKEN }}
 
       - run: echo "SOURCE=$(make echo-source)" >> $GITHUB_ENV
       - run: echo "DATABASE=$(make echo-database)" >> $GITHUB_ENV
 
-      - uses: goreleaser/goreleaser-action@v2
+      - uses: goreleaser/goreleaser-action@v5
         with:
           version: latest
           args: release --rm-dist
@@ -94,9 +94,15 @@ jobs:
       - run: package_cloud push golang-migrate/migrate/ubuntu/focal dist/migrate.linux-amd64.deb
         env:
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - run: package_cloud push golang-migrate/migrate/ubuntu/jammy dist/migrate.linux-amd64.deb
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
       - run: package_cloud push golang-migrate/migrate/debian/buster dist/migrate.linux-amd64.deb
         env:
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
       - run: package_cloud push golang-migrate/migrate/debian/bullseye dist/migrate.linux-amd64.deb
         env:
           PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}
+      - run: package_cloud push golang-migrate/migrate/debian/bookworm dist/migrate.linux-amd64.deb
+        env:
+          PACKAGECLOUD_TOKEN: ${{ secrets.PACKAGECLOUD_TOKEN }}

.goreleaser.yml

@@ -87,6 +87,7 @@ release:
   prerelease: auto
 source:
   enabled: true
+  rlcp: true
   format: zip
 changelog:
   skip: false

Dockerfile

@@ -1,4 +1,4 @@
-FROM golang:1.19-alpine3.16 AS builder
+FROM golang:1.21-alpine3.19 AS builder
 ARG VERSION
 
 RUN apk add --no-cache git gcc musl-dev make
@@ -15,7 +15,7 @@ COPY . ./
 
 RUN make build-docker
 
-FROM alpine:3.16
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

Dockerfile.github-actions

@@ -1,4 +1,4 @@
-FROM alpine:3.16
+FROM alpine:3.19
 
 RUN apk add --no-cache ca-certificates
 

FAQ.md

@@ -50,7 +50,7 @@
   and whenever we want, not just once at the beginning of all tests.
 
 #### Can I maintain my driver in my own repository?
-  Yes, technically thats possible. We want to encourage you to contribute your driver to this respository though.
+  Yes, technically thats possible. We want to encourage you to contribute your driver to this repository though.
   The driver's functionality is dictated by migrate's interfaces. That means there should really
   just be one driver for a database/ source. We want to prevent a future where several drivers doing the exact same thing,
   just implemented a bit differently, co-exist somewhere on GitHub. If users have to do research first to find the

GETTING_STARTED.md

@@ -28,7 +28,7 @@ migrate -database YOUR_DATABASE_URL -path PATH_TO_YOUR_MIGRATIONS up
 
 Just add the code to your app and you're ready to go!
 
-Before commiting your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
+Before committing your migrations you should run your migrations up, down, and then up again to see if migrations are working properly both ways.
 (e.g. if you created a table in a migration but reverse migration did not delete it, you will encounter an error when running the forward migration again)
 It's also worth checking your migrations in a separate, containerized environment. You can find some tools at the [end of this document](#further-reading).
 

Makefile

@@ -1,5 +1,5 @@
 SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab
-DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx
+DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite
 DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher
 VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-)
 TEST_FLAGS ?=

README.md

@@ -1,9 +1,9 @@
-[![GitHub Workflow Status (branch)](https://img.shields.io/github/workflow/status/golang-migrate/migrate/CI/master)](https://github.com/golang-migrate/migrate/actions/workflows/ci.yaml?query=branch%3Amaster)
+[![GitHub Workflow Status (branch)](https://img.shields.io/github/actions/workflow/status/golang-migrate/migrate/ci.yaml?branch=master)](https://github.com/golang-migrate/migrate/actions/workflows/ci.yaml?query=branch%3Amaster)
 [![GoDoc](https://pkg.go.dev/badge/github.com/golang-migrate/migrate)](https://pkg.go.dev/github.com/golang-migrate/migrate/v4)
 [![Coverage Status](https://img.shields.io/coveralls/github/golang-migrate/migrate/master.svg)](https://coveralls.io/github/golang-migrate/migrate?branch=master)
 [![packagecloud.io](https://img.shields.io/badge/deb-packagecloud.io-844fec.svg)](https://packagecloud.io/golang-migrate/migrate?filter=debs)
 [![Docker Pulls](https://img.shields.io/docker/pulls/migrate/migrate.svg)](https://hub.docker.com/r/migrate/migrate/)
-![Supported Go Versions](https://img.shields.io/badge/Go-1.18%2C%201.19-lightgrey.svg)
+![Supported Go Versions](https://img.shields.io/badge/Go-1.19%2C%201.20-lightgrey.svg)
 [![GitHub Release](https://img.shields.io/github/release/golang-migrate/migrate.svg)](https://github.com/golang-migrate/migrate/releases)
 [![Go Report Card](https://goreportcard.com/badge/github.com/golang-migrate/migrate/v4)](https://goreportcard.com/report/github.com/golang-migrate/migrate/v4)
 
@@ -24,14 +24,15 @@ Forked from [mattes/migrate](https://github.com/mattes/migrate)
 Database drivers run migrations. [Add a new database?](database/driver.go)
 
 * [PostgreSQL](database/postgres)
-* [PGX](database/pgx)
+* [PGX v4](database/pgx)
+* [PGX v5](database/pgx/v5)
 * [Redshift](database/redshift)
 * [Ql](database/ql)
-* [Cassandra](database/cassandra)
+* [Cassandra / ScyllaDB](database/cassandra)
 * [SQLite](database/sqlite)
 * [SQLite3](database/sqlite3) ([todo #165](https://github.com/mattes/migrate/issues/165))
 * [SQLCipher](database/sqlcipher)
-* [MySQL/ MariaDB](database/mysql)
+* [MySQL / MariaDB](database/mysql)
 * [Neo4j](database/neo4j)
 * [MongoDB](database/mongodb)
 * [CrateDB](database/crate) ([todo #170](https://github.com/mattes/migrate/issues/170))
@@ -42,6 +43,7 @@ Database drivers run migrations. [Add a new database?](database/driver.go)
 * [ClickHouse](database/clickhouse)
 * [Firebird](database/firebird)
 * [MS SQL Server](database/sqlserver)
+* [RQLite](database/rqlite)
 
 ### Database URLs
 
@@ -169,6 +171,11 @@ Each migration has an up and down migration. [Why?](FAQ.md#why-two-separate-file
 
 [Best practices: How to write migrations.](MIGRATIONS.md)
 
+## Coming from another db migration tool?
+
+Check out [migradaptor](https://github.com/musinit/migradaptor/).
+*Note: migradaptor is not affliated or supported by this project*
+
 ## Versions
 
 Version | Supported? | Import | Notes

cmd/migrate/README.md

@@ -7,7 +7,7 @@
 [Release Downloads](https://github.com/golang-migrate/migrate/releases)
 
 ```bash
-$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$platform-amd64.tar.gz | tar xvz
+$ curl -L https://github.com/golang-migrate/migrate/releases/download/$version/migrate.$os-$arch.tar.gz | tar xvz
 ```
 
 ### MacOS

database/cassandra/README.md

@@ -1,12 +1,17 @@
-# Cassandra
+# Cassandra / ScyllaDB
 
-* Drop command will not work on Cassandra 2.X because it rely on
+* `Drop()` method will not work on Cassandra 2.X because it rely on
 system_schema table which comes with 3.X
-* Other commands should work properly but are **not tested**
+* Other methods should work properly but are **not tested**
 * The Cassandra driver (gocql) does not natively support executing multiple statements in a single query. To allow for multiple statements in a single migration, you can use the `x-multi-statement` param. There are two important caveats:
   * This mode splits the migration text into separately-executed statements by a semi-colon `;`. Thus `x-multi-statement` cannot be used when a statement in the migration contains a string with a semi-colon.
   * The queries are not executed in any sort of transaction/batch, meaning you are responsible for fixing partial migrations.
 
+**ScyllaDB**
+
+* No additional configuration is required since it is a drop-in replacement for Cassandra.
+* The `Drop()` method` works for ScyllaDB 5.1
+
 
 ## Usage
 `cassandra://host:port/keyspace?param1=value&param2=value2`

database/cassandra/cassandra_test.go

@@ -25,9 +25,11 @@ var (
 	// Although Cassandra 2.x is supported by the Apache Foundation,
 	// the migrate db driver only supports Cassandra 3.x since it uses
 	// the system_schema keyspace.
+	// last ScyllaDB version tested is 5.1.11
 	specs = []dktesting.ContainerSpec{
 		{ImageName: "cassandra:3.0", Options: opts},
 		{ImageName: "cassandra:3.11", Options: opts},
+		{ImageName: "scylladb/scylla:5.1.11", Options: opts},
 	}
 )
 

database/clickhouse/clickhouse.go

@@ -220,7 +220,7 @@ func (ch *ClickHouse) ensureVersionTable() (err error) {
 
 	var (
 		table string
-		query = "SHOW TABLES FROM " + ch.config.DatabaseName + " LIKE '" + ch.config.MigrationsTable + "'"
+		query = "SHOW TABLES FROM " + quoteIdentifier(ch.config.DatabaseName) + " LIKE '" + ch.config.MigrationsTable + "'"
 	)
 	// check if migration table exists
 	if err := ch.conn.QueryRow(query).Scan(&table); err != nil {
@@ -259,7 +259,7 @@ func (ch *ClickHouse) ensureVersionTable() (err error) {
 }
 
 func (ch *ClickHouse) Drop() (err error) {
-	query := "SHOW TABLES FROM " + ch.config.DatabaseName
+	query := "SHOW TABLES FROM " + quoteIdentifier(ch.config.DatabaseName)
 	tables, err := ch.conn.Query(query)
 
 	if err != nil {
@@ -277,7 +277,7 @@ func (ch *ClickHouse) Drop() (err error) {
 			return err
 		}
 
-		query = "DROP TABLE IF EXISTS " + ch.config.DatabaseName + "." + table
+		query = "DROP TABLE IF EXISTS " + quoteIdentifier(ch.config.DatabaseName) + "." + quoteIdentifier(table)
 
 		if _, err := ch.conn.Exec(query); err != nil {
 			return &database.Error{OrigErr: err, Query: []byte(query)}
@@ -305,3 +305,12 @@ func (ch *ClickHouse) Unlock() error {
 	return nil
 }
 func (ch *ClickHouse) Close() error { return ch.conn.Close() }
+
+// Copied from lib/pq implementation: https://github.com/lib/pq/blob/v1.9.0/conn.go#L1611
+func quoteIdentifier(name string) string {
+	end := strings.IndexRune(name, 0)
+	if end > -1 {
+		name = name[:end]
+	}
+	return `"` + strings.Replace(name, `"`, `""`, -1) + `"`
+}

database/parse_test.go

@@ -8,13 +8,15 @@ import (
 )
 
 const reservedChars = "!#$%&'()*+,/:;=?@[]"
+const reservedCharTestNamePrefix = "reserved char "
 
 const baseUsername = "username"
 
+const scheme = "database://"
+
 // TestUserUnencodedReservedURLChars documents the behavior of using unencoded reserved characters in usernames with
 // net/url Parse()
 func TestUserUnencodedReservedURLChars(t *testing.T) {
-	scheme := "database://"
 	urlSuffix := "password@localhost:12345/myDB?someParam=true"
 	urlSuffixAndSep := ":" + urlSuffix
 
@@ -64,7 +66,7 @@ func TestUserUnencodedReservedURLChars(t *testing.T) {
 	testedChars := make([]string, 0, len(reservedChars))
 	for _, tc := range testcases {
 		testedChars = append(testedChars, tc.char)
-		t.Run("reserved char "+tc.char, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+tc.char, func(t *testing.T) {
 			s := scheme + baseUsername + tc.char + urlSuffixAndSep
 			u, err := url.Parse(s)
 			if err == nil {
@@ -98,13 +100,12 @@ func TestUserUnencodedReservedURLChars(t *testing.T) {
 }
 
 func TestUserEncodedReservedURLChars(t *testing.T) {
-	scheme := "database://"
 	urlSuffix := "password@localhost:12345/myDB?someParam=true"
 	urlSuffixAndSep := ":" + urlSuffix
 
 	for _, c := range reservedChars {
 		c := string(c)
-		t.Run("reserved char "+c, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+c, func(t *testing.T) {
 			encodedChar := "%" + hex.EncodeToString([]byte(c))
 			s := scheme + baseUsername + encodedChar + urlSuffixAndSep
 			expectedUsername := baseUsername + c
@@ -126,7 +127,7 @@ func TestUserEncodedReservedURLChars(t *testing.T) {
 // with net/url Parse()
 func TestPasswordUnencodedReservedURLChars(t *testing.T) {
 	username := baseUsername
-	schemeAndUsernameAndSep := "database://" + username + ":"
+	schemeAndUsernameAndSep := scheme + username + ":"
 	basePassword := "password"
 	urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
 
@@ -174,7 +175,7 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) {
 	testedChars := make([]string, 0, len(reservedChars))
 	for _, tc := range testcases {
 		testedChars = append(testedChars, tc.char)
-		t.Run("reserved char "+tc.char, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+tc.char, func(t *testing.T) {
 			s := schemeAndUsernameAndSep + basePassword + tc.char + urlSuffixAndSep
 			u, err := url.Parse(s)
 			if err == nil {
@@ -213,13 +214,13 @@ func TestPasswordUnencodedReservedURLChars(t *testing.T) {
 
 func TestPasswordEncodedReservedURLChars(t *testing.T) {
 	username := baseUsername
-	schemeAndUsernameAndSep := "database://" + username + ":"
+	schemeAndUsernameAndSep := scheme + username + ":"
 	basePassword := "password"
 	urlSuffixAndSep := "@localhost:12345/myDB?someParam=true"
 
 	for _, c := range reservedChars {
 		c := string(c)
-		t.Run("reserved char "+c, func(t *testing.T) {
+		t.Run(reservedCharTestNamePrefix+c, func(t *testing.T) {
 			encodedChar := "%" + hex.EncodeToString([]byte(c))
 			s := schemeAndUsernameAndSep + basePassword + encodedChar + urlSuffixAndSep
 			expectedPassword := basePassword + c

database/pgx/README.md

@@ -1,5 +1,7 @@
 # pgx
 
+This package is for [pgx/v4](https://pkg.go.dev/github.com/jackc/pgx/v4). A backend for the newer [pgx/v5](https://pkg.go.dev/github.com/jackc/pgx/v5) is [also available](v5).
+
 `pgx://user:password@host:port/dbname?query`
 
 | URL Query  | WithInstance Config | Description |

database/pgx/pgx.go

@@ -28,6 +28,7 @@ import (
 func init() {
 	db := Postgres{}
 	database.Register("pgx", &db)
+	database.Register("pgx4", &db)
 }
 
 var (
@@ -150,7 +151,7 @@ func (p *Postgres) Open(url string) (database.Driver, error) {
 	// i.e. pgx://user:password@host:port/db => postgres://user:password@host:port/db
 	purl.Scheme = "postgres"
 
-	db, err := sql.Open("pgx", migrate.FilterCustomQuery(purl).String())
+	db, err := sql.Open("pgx/v4", migrate.FilterCustomQuery(purl).String())
 	if err != nil {
 		return nil, err
 	}

database/pgx/pgx_test.go

@@ -8,19 +8,17 @@ import (
 	sqldriver "database/sql/driver"
 	"errors"
 	"fmt"
-	"log"
-
 	"io"
+	"log"
 	"strconv"
 	"strings"
 	"sync"
 	"testing"
 
-	"github.com/golang-migrate/migrate/v4"
-
 	"github.com/dhui/dktest"
-
+	"github.com/golang-migrate/migrate/v4"
 	"github.com/golang-migrate/migrate/v4/database"
+
 	dt "github.com/golang-migrate/migrate/v4/database/testing"
 	"github.com/golang-migrate/migrate/v4/dktesting"
 	_ "github.com/golang-migrate/migrate/v4/source/file"
@@ -41,6 +39,9 @@ var (
 		{ImageName: "postgres:10", Options: opts},
 		{ImageName: "postgres:11", Options: opts},
 		{ImageName: "postgres:12", Options: opts},
+		{ImageName: "postgres:13", Options: opts},
+		{ImageName: "postgres:14", Options: opts},
+		{ImageName: "postgres:15", Options: opts},
 	}
 )