diff --git a/Makefile b/Makefile index 8e23a43c7..71c7c21a9 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ SOURCE ?= file go_bindata github github_ee bitbucket aws_s3 google_cloud_storage godoc_vfs gitlab DATABASE ?= postgres mysql redshift cassandra spanner cockroachdb yugabytedb clickhouse mongodb sqlserver firebird neo4j pgx pgx5 rqlite -DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher +DATABASE_TEST ?= $(DATABASE) sqlite sqlite3 sqlcipher oracle VERSION ?= $(shell git describe --tags 2>/dev/null | cut -c 2-) TEST_FLAGS ?= REPO_OWNER ?= $(shell cd .. && basename "$$(pwd)") diff --git a/README.md b/README.md index a79cc7b76..5980c9bc0 100644 --- a/README.md +++ b/README.md @@ -44,7 +44,8 @@ Database drivers run migrations. [Add a new database?](database/driver.go) * [Firebird](database/firebird) * [MS SQL Server](database/sqlserver) * [rqlite](database/rqlite) - +* [Oracle](database/oracle) +* ### Database URLs Database connection strings are specified via URLs. The URL format is driver dependent but generally has the form: `dbdriver://username:password@host:port/dbname?param1=true¶m2=false` diff --git a/database/oracle/README.md b/database/oracle/README.md new file mode 100644 index 000000000..717d04871 --- /dev/null +++ b/database/oracle/README.md @@ -0,0 +1,110 @@ +# oracle + +The supported oracle specific options can be configured in the query section of the oracle +URL `oracle://user:password@host:port/ServiceName?query` + +| URL Query | WithInstance Config | Description | +|--------------------------|----------------------|-------------------------------------------------------------------------------------------------------------------------| +| `x-migrations-table` | `MigrationsTable` | Name of the migrations table in UPPER case | +| `x-multi-stmt-enabled` | `MultiStmtEnabled` | If the migration files are in multi-statements style | +| `x-multi-stmt-separator` | `MultiStmtSeparator` | a single line which use as the token to spilt multiple statements in single migration file, triple-dash separator `---` | + +## Write migration files + +There are two ways to write the migration files, + +1. Single statement file in which it contains only one SQL statement or one PL/SQL statement(Default) +2. Multi statements file in which it can have multi statements(can be SQL or PL/SQL or mixed) + +### Single statement file + +Oracle godor driver support process one statement at a time, so it is natural to support single statement per file as +the default. +Check the [single statement migration files](examples/migrations) as an example. + +### Multi statements file + +Although the golang oracle driver [godror](https://github.com/godror/godror) does not natively support executing +multiple +statements in a single query, it's more friendly and handy to support multi statements in a single migration file in +some case, +so the multi statements can be separated with a line separator(default to triple-dash separator ---), for example: + +``` +statement 1 +--- +statement 2 +``` + +Check the [multi statements' migration files](examples/migrations-multistmt) as an example. + +## Supported & tested version + +- 18-xe + +## Build cli + +```bash +$ cd /path/to/repo/dir +$ go build -tags 'oracle' -o bin/migrate github.com/golang-migrate/migrate/v4/cli +``` + +## Run test code + +There are two ways to run the test code: + +- Run the test code locally with an existing Oracle Instance(Recommended) +- Run the test code inside a container just like CI, It will require to start an Oracle container every time, and it's + very time expense. + +### Run the test code locally with an existing Oracle Instance + +1. Start the `Oracle Database Instance` via docker first, so that you can reuse whenever you want to run the test code. + +```bash +$ cat docker-compose.yaml +--- +services: + oracle-db: + container_name: oracle-db + image: gvenzl/oracle-free:23.5-slim + environment: + ORACLE_PASSWORD: SuperPassword@2025 + ports: + - 1521:1521 + healthcheck: + test: ["CMD", "healthcheck.sh"] + interval: 10s + timeout: 5s + retries: 10 + start_period: 5s + start_interval: 5s + volumes: + - ${HOME}/database/oracle/testdata/init.sql:/docker-entrypoint-initdb.d/init.sql +``` + +2. Go into the sqlplus console + +```bash +$ docker exec -it orclxe bash +# su oracle +$ sqlplus / as sysdba +``` + +3. Create a test DB + +```sql +alter session set container=FREEPDB1; +create user orcl identified by orcl; +grant dba to orcl; +grant create session to orcl; +grant connect, resource to orcl; +grant all privileges to orcl; +``` + +4. Run the test code + +```bash +$ cd /path/to/repo/database/oracle/dir +$ ORACLE_DSN=oracle://orcl:orcl@localhost:1521/FREEPDB1 go test -tags "oracle" -race -v -covermode atomic ./... -coverprofile .coverage -timeout 20m +``` \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.down.sql b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..20f9ae508 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS USERS_MS; +--- \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.up.sql b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..49235b96c --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1085649617_create_users_table.up.sql @@ -0,0 +1,17 @@ +CREATE TABLE USERS_MS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); + +--- + +DROP TABLE IF EXISTS USERS_MS; + +--- + +CREATE TABLE USERS_MS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.down.sql b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..c0336477f --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE USERS_MS DROP COLUMN CITY; \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.up.sql b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..468e508ee --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1185749658_add_city_to_users.up.sql @@ -0,0 +1,3 @@ +ALTER TABLE USERS_MS ADD CITY varchar(100); +--- +ALTER TABLE USERS_MS ADD ALIAS varchar(100); \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.down.sql b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..5b946947d --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1,2 @@ +DROP INDEX users_ms_email_index; +--- \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.up.sql b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..6af1bbee5 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX users_ms_email_index ON users_ms (email); \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.down.sql b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..aa5932d25 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.down.sql @@ -0,0 +1,2 @@ +DROP TABLE IF EXISTS BOOKS_MS; +--- \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.up.sql b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..16561c077 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE BOOKS_MS ( + USER_ID integer, + NAME varchar(40), + AUTHOR varchar(40) +); \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.down.sql b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..55965f68c --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS MOVIES_MS \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.up.sql b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..56a8cb974 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE MOVIES_MS ( + USER_ID integer, + NAME varchar(40), + DIRECTOR varchar(40) +); \ No newline at end of file diff --git a/database/oracle/examples/migrations-multistmt/1585849751_just_a_comment.up.sql b/database/oracle/examples/migrations-multistmt/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations-multistmt/1685849751_another_comment.up.sql b/database/oracle/examples/migrations-multistmt/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations-multistmt/1785849751_another_comment.up.sql b/database/oracle/examples/migrations-multistmt/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations-multistmt/1885849751_another_comment.up.sql b/database/oracle/examples/migrations-multistmt/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations-multistmt/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1085649617_create_users_table.down.sql b/database/oracle/examples/migrations/1085649617_create_users_table.down.sql new file mode 100644 index 000000000..82b91296f --- /dev/null +++ b/database/oracle/examples/migrations/1085649617_create_users_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS USERS \ No newline at end of file diff --git a/database/oracle/examples/migrations/1085649617_create_users_table.up.sql b/database/oracle/examples/migrations/1085649617_create_users_table.up.sql new file mode 100644 index 000000000..76ed60b1e --- /dev/null +++ b/database/oracle/examples/migrations/1085649617_create_users_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +) \ No newline at end of file diff --git a/database/oracle/examples/migrations/1185749658_add_city_to_users.down.sql b/database/oracle/examples/migrations/1185749658_add_city_to_users.down.sql new file mode 100644 index 000000000..27ecfb15e --- /dev/null +++ b/database/oracle/examples/migrations/1185749658_add_city_to_users.down.sql @@ -0,0 +1 @@ +ALTER TABLE USERS DROP COLUMN CITY \ No newline at end of file diff --git a/database/oracle/examples/migrations/1185749658_add_city_to_users.up.sql b/database/oracle/examples/migrations/1185749658_add_city_to_users.up.sql new file mode 100644 index 000000000..7c4ecf46a --- /dev/null +++ b/database/oracle/examples/migrations/1185749658_add_city_to_users.up.sql @@ -0,0 +1 @@ +ALTER TABLE USERS ADD CITY varchar(100) \ No newline at end of file diff --git a/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.down.sql b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.down.sql new file mode 100644 index 000000000..8cd5403af --- /dev/null +++ b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.down.sql @@ -0,0 +1 @@ +DROP INDEX IF EXISTS users_email_index \ No newline at end of file diff --git a/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.up.sql b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.up.sql new file mode 100644 index 000000000..3b09b427e --- /dev/null +++ b/database/oracle/examples/migrations/1285849751_add_index_on_user_emails.up.sql @@ -0,0 +1 @@ +CREATE UNIQUE INDEX users_email_index ON users (email) \ No newline at end of file diff --git a/database/oracle/examples/migrations/1385949617_create_books_table.down.sql b/database/oracle/examples/migrations/1385949617_create_books_table.down.sql new file mode 100644 index 000000000..0f6d91b92 --- /dev/null +++ b/database/oracle/examples/migrations/1385949617_create_books_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS BOOKS \ No newline at end of file diff --git a/database/oracle/examples/migrations/1385949617_create_books_table.up.sql b/database/oracle/examples/migrations/1385949617_create_books_table.up.sql new file mode 100644 index 000000000..797915e72 --- /dev/null +++ b/database/oracle/examples/migrations/1385949617_create_books_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE BOOKS ( + USER_ID integer, + NAME varchar(40), + AUTHOR varchar(40) +) \ No newline at end of file diff --git a/database/oracle/examples/migrations/1485949617_create_movies_table.down.sql b/database/oracle/examples/migrations/1485949617_create_movies_table.down.sql new file mode 100644 index 000000000..9bc5bb609 --- /dev/null +++ b/database/oracle/examples/migrations/1485949617_create_movies_table.down.sql @@ -0,0 +1 @@ +DROP TABLE IF EXISTS MOVIES \ No newline at end of file diff --git a/database/oracle/examples/migrations/1485949617_create_movies_table.up.sql b/database/oracle/examples/migrations/1485949617_create_movies_table.up.sql new file mode 100644 index 000000000..5f8f7c74c --- /dev/null +++ b/database/oracle/examples/migrations/1485949617_create_movies_table.up.sql @@ -0,0 +1,5 @@ +CREATE TABLE MOVIES ( + USER_ID integer, + NAME varchar(40), + DIRECTOR varchar(40) +) \ No newline at end of file diff --git a/database/oracle/examples/migrations/1585849751_just_a_comment.up.sql b/database/oracle/examples/migrations/1585849751_just_a_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1585849751_just_a_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1685849751_another_comment.up.sql b/database/oracle/examples/migrations/1685849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1685849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1785849751_another_comment.up.sql b/database/oracle/examples/migrations/1785849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1785849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/examples/migrations/1885849751_another_comment.up.sql b/database/oracle/examples/migrations/1885849751_another_comment.up.sql new file mode 100644 index 000000000..9b6b57a61 --- /dev/null +++ b/database/oracle/examples/migrations/1885849751_another_comment.up.sql @@ -0,0 +1 @@ +-- Lorem ipsum dolor sit amet, consectetur adipiscing elit. Aenean sed interdum velit, tristique iaculis justo. Pellentesque ut porttitor dolor. Donec sit amet pharetra elit. Cras vel ligula ex. Phasellus posuere. diff --git a/database/oracle/oracle.go b/database/oracle/oracle.go new file mode 100644 index 000000000..c5d8e1752 --- /dev/null +++ b/database/oracle/oracle.go @@ -0,0 +1,455 @@ +package oracle + +import ( + "bufio" + "bytes" + "context" + "database/sql" + "fmt" + "io" + nurl "net/url" + "strconv" + "strings" + + "github.com/golang-migrate/migrate/v4" + "github.com/golang-migrate/migrate/v4/database" + "github.com/hashicorp/go-multierror" + _ "github.com/sijms/go-ora/v2" +) + +func init() { + db := Oracle{} + database.Register("oracle", &db) +} + +const ( + migrationsTableQueryKey = "x-migrations-table" + multiStmtEnableQueryKey = "x-multi-stmt-enabled" + multiStmtSeparatorQueryKey = "x-multi-stmt-separator" +) + +var ( + DefaultMigrationsTable = "SCHEMA_MIGRATIONS" + DefaultMultiStmtEnabled = false + DefaultMultiStmtSeparator = "---" +) + +var ( + ErrNilConfig = fmt.Errorf("no config") + ErrNoDatabaseName = fmt.Errorf("no database name") +) + +type Config struct { + MigrationsTable string + MultiStmtEnabled bool + MultiStmtSeparator string + + databaseName string +} + +type Oracle struct { + // Locking and unlocking need to use the same connection + conn *sql.Conn + db *sql.DB + isLocked bool + + // Open and WithInstance need to guarantee that config is never nil + config *Config +} + +func WithInstance(instance *sql.DB, config *Config) (database.Driver, error) { + if config == nil { + return nil, ErrNilConfig + } + + if err := instance.Ping(); err != nil { + return nil, err + } + + query := `SELECT SYS_CONTEXT('USERENV','DB_NAME') FROM DUAL` + var dbName string + if err := instance.QueryRow(query).Scan(&dbName); err != nil { + return nil, &database.Error{OrigErr: err, Query: []byte(query)} + } + + if dbName == "" { + return nil, ErrNoDatabaseName + } + + config.databaseName = dbName + + if config.MigrationsTable == "" { + config.MigrationsTable = DefaultMigrationsTable + } + + if config.MultiStmtSeparator == "" { + config.MultiStmtSeparator = DefaultMultiStmtSeparator + } + + conn, err := instance.Conn(context.Background()) + + if err != nil { + return nil, err + } + + ora := &Oracle{ + conn: conn, + db: instance, + config: config, + } + + if err := ora.ensureVersionTable(); err != nil { + return nil, err + } + + return ora, nil +} + +func (ora *Oracle) Open(url string) (database.Driver, error) { + purl, err := nurl.Parse(url) + if err != nil { + return nil, err + } + + db, err := sql.Open("oracle", migrate.FilterCustomQuery(purl).String()) + if err != nil { + return nil, err + } + + migrationsTable := DefaultMigrationsTable + if s := purl.Query().Get(migrationsTableQueryKey); len(s) > 0 { + migrationsTable = strings.ToUpper(s) + } + + multiStmtEnabled := DefaultMultiStmtEnabled + if s := purl.Query().Get(multiStmtEnableQueryKey); len(s) > 0 { + multiStmtEnabled, err = strconv.ParseBool(s) + if err != nil { + return nil, fmt.Errorf("unable to parse option %s: %w", multiStmtEnableQueryKey, err) + } + } + + multiStmtSeparator := DefaultMultiStmtSeparator + if s := purl.Query().Get(multiStmtSeparatorQueryKey); len(s) > 0 { + multiStmtSeparator = s + } + + oraInst, err := WithInstance(db, &Config{ + databaseName: purl.Path, + MigrationsTable: migrationsTable, + MultiStmtEnabled: multiStmtEnabled, + MultiStmtSeparator: multiStmtSeparator, + }) + + if err != nil { + return nil, err + } + + return oraInst, nil +} + +func (ora *Oracle) Close() error { + connErr := ora.conn.Close() + dbErr := ora.db.Close() + if connErr != nil || dbErr != nil { + return fmt.Errorf("conn: %v, db: %v", connErr, dbErr) + } + return nil +} + +func (ora *Oracle) Lock() error { + if ora.isLocked { + return database.ErrLocked + } + + // https://docs.oracle.com/cd/B28359_01/appdev.111/b28419/d_lock.htm#ARPLS021 + query := ` +declare + v_lockhandle varchar2(200); + v_result number; +begin + dbms_lock.allocate_unique('control_lock', v_lockhandle); + v_result := dbms_lock.request(v_lockhandle, dbms_lock.x_mode); + if v_result <> 0 then + dbms_output.put_line( + case + when v_result=1 then 'Timeout' + when v_result=2 then 'Deadlock' + when v_result=3 then 'Parameter Error' + when v_result=4 then 'Already owned' + when v_result=5 then 'Illegal Lock Handle' + end); + end if; +end; +` + if _, err := ora.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Err: "try lock failed", Query: []byte(query)} + } + + ora.isLocked = true + return nil +} + +func (ora *Oracle) Unlock() error { + if !ora.isLocked { + return nil + } + + query := ` +declare + v_lockhandle varchar2(200); + v_result number; +begin + dbms_lock.allocate_unique('control_lock', v_lockhandle); + v_result := dbms_lock.release(v_lockhandle); + if v_result <> 0 then + dbms_output.put_line( + case + when v_result=1 then 'Timeout' + when v_result=2 then 'Deadlock' + when v_result=3 then 'Parameter Error' + when v_result=4 then 'Already owned' + when v_result=5 then 'Illegal Lock Handle' + end); + end if; +end; +` + if _, err := ora.conn.ExecContext(context.Background(), query); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + ora.isLocked = false + return nil +} + +func (ora *Oracle) Run(migration io.Reader) error { + var queries []string + if !ora.config.MultiStmtEnabled { + // If multi-statements is not enabled explicitly, + // i.e, there is no multi-statement enabled(neither normal multi-statements nor multi-PL/SQL-statements), + // consider the whole migration as a blob. + query, err := removeComments(migration) + if err != nil { + return err + } + if query == "" { + // empty query, do nothing + return nil + } + queries = append(queries, query) + } else { + // If multi-statements is enabled explicitly, + // there could be multi-statements or multi-PL/SQL-statements in a single migration. + var err error + queries, err = parseMultiStatements(migration, ora.config.MultiStmtSeparator) + if err != nil { + return err + } + } + + for _, query := range queries { + if _, err := ora.conn.ExecContext(context.Background(), query); err != nil { + return database.Error{OrigErr: err, Err: "migration failed", Query: []byte(query)} + } + } + + return nil +} + +func (ora *Oracle) SetVersion(version int, dirty bool) error { + tx, err := ora.conn.BeginTx(context.Background(), &sql.TxOptions{}) + if err != nil { + return &database.Error{OrigErr: err, Err: "transaction start failed"} + } + + query := "TRUNCATE TABLE " + ora.config.MigrationsTable + if _, err := tx.Exec(query); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + if version >= 0 || (version == database.NilVersion && dirty) { + query = `INSERT INTO ` + ora.config.MigrationsTable + ` (VERSION, DIRTY) VALUES (:1, :2)` + if _, err := tx.Exec(query, version, b2i(dirty)); err != nil { + if errRollback := tx.Rollback(); errRollback != nil { + err = multierror.Append(err, errRollback) + } + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + + if err := tx.Commit(); err != nil { + return &database.Error{OrigErr: err, Err: "transaction commit failed"} + } + + return nil +} + +func (ora *Oracle) Version() (version int, dirty bool, err error) { + query := "SELECT VERSION, DIRTY FROM " + ora.config.MigrationsTable + " WHERE ROWNUM = 1 ORDER BY VERSION desc" + err = ora.conn.QueryRowContext(context.Background(), query).Scan(&version, &dirty) + switch { + case err == sql.ErrNoRows: + return database.NilVersion, false, nil + case err != nil: + return 0, false, &database.Error{OrigErr: err, Query: []byte(query)} + default: + return version, dirty, nil + } +} + +func (ora *Oracle) Drop() (err error) { + // select all tables in current schema + query := `SELECT TABLE_NAME FROM USER_TABLES` + tables, err := ora.conn.QueryContext(context.Background(), query) + if err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + defer func() { + if errClose := tables.Close(); errClose != nil { + err = multierror.Append(err, errClose) + } + }() + + // delete one table after another + tableNames := make([]string, 0) + for tables.Next() { + var tableName string + if err := tables.Scan(&tableName); err != nil { + return err + } + if len(tableName) > 0 { + tableNames = append(tableNames, tableName) + } + } + + query = `DROP TABLE %s CASCADE CONSTRAINTS` + if len(tableNames) > 0 { + // delete one by one ... + for _, t := range tableNames { + if _, err := ora.conn.ExecContext(context.Background(), fmt.Sprintf(query, t)); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + } + } + + return nil +} + +// ensureVersionTable checks if versions table exists and, if not, creates it. +// Note that this function locks the database, which deviates from the usual +// convention of "caller locks" in the Postgres type. +func (ora *Oracle) ensureVersionTable() (err error) { + if err = ora.Lock(); err != nil { + return err + } + + defer func() { + if e := ora.Unlock(); e != nil { + if err == nil { + err = e + } else { + err = multierror.Append(err, e) + } + } + }() + + query := ` +declare +v_sql LONG; +begin +v_sql:='create table %s + ( + VERSION NUMBER(20) NOT NULL PRIMARY KEY, + DIRTY NUMBER(1) NOT NULL + )'; +execute immediate v_sql; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE = -955 THEN + NULL; -- suppresses ORA-00955 exception + ELSE + RAISE; + END IF; +END; +` + if _, err = ora.conn.ExecContext(context.Background(), fmt.Sprintf(query, ora.config.MigrationsTable)); err != nil { + return &database.Error{OrigErr: err, Query: []byte(query)} + } + + return nil +} + +func b2i(b bool) int { + if b { + return 1 + } + return 0 +} + +func removeComments(rd io.Reader) (string, error) { + buf := bytes.Buffer{} + scanner := bufio.NewScanner(rd) + for scanner.Scan() { + line := scanner.Text() + // ignore comment + if strings.HasPrefix(line, "--") { + continue + } + if _, err := buf.WriteString(line + "\n"); err != nil { + return "", err + } + } + return buf.String(), nil +} + +func parseMultiStatements(rd io.Reader, plsqlStmtSeparator string) ([]string, error) { + var results []string + var buf bytes.Buffer + scanner := bufio.NewScanner(rd) + for scanner.Scan() { + line := scanner.Text() + if line == plsqlStmtSeparator { + results = append(results, buf.String()) + buf.Reset() + continue + } + if line == "" || strings.HasPrefix(line, "--") { + continue // ignore empty and comment line + } + if _, err := buf.WriteString(line + "\n"); err != nil { + return nil, err + } + } + if buf.Len() > 0 { + // append the final result if it's not empty + results = append(results, buf.String()) + } + + queries := make([]string, 0, len(results)) + for _, result := range results { + result = strings.TrimSpace(result) + result = strings.TrimPrefix(result, "\n") + result = strings.TrimSuffix(result, "\n") + if !isPLSQLTail(result) { + // remove the ";" from the tail if it's not PL/SQL stmt + result = strings.TrimSuffix(result, ";") + } + if result == "" { + continue // skip empty query + } + queries = append(queries, result) + } + return queries, nil +} + +func isPLSQLTail(s string) bool { + plsqlTail := "end;" + if len(s) < len(plsqlTail) { + return false + } + pos := len(s) - len(plsqlTail) + tail := s[pos:] + return strings.EqualFold(tail, plsqlTail) +} diff --git a/database/oracle/oracle_test.go b/database/oracle/oracle_test.go new file mode 100644 index 000000000..8b521aa51 --- /dev/null +++ b/database/oracle/oracle_test.go @@ -0,0 +1,270 @@ +package oracle + +import ( + "bytes" + "context" + "database/sql" + sqldriver "database/sql/driver" + "fmt" + "io" + "log" + "os" + "path/filepath" + "testing" + + "github.com/dhui/dktest" + "github.com/docker/docker/api/types/mount" + "github.com/docker/go-connections/nat" + "github.com/golang-migrate/migrate/v4" + dt "github.com/golang-migrate/migrate/v4/database/testing" + "github.com/golang-migrate/migrate/v4/dktesting" + _ "github.com/golang-migrate/migrate/v4/source/file" + "github.com/stretchr/testify/require" + "github.com/stretchr/testify/suite" +) + +const ( + defaultPort = 1521 + userdba = "orcl" + userdbaPass = "orcl" + defaultPass = "orcl" +) + +var ( + specs = []dktesting.ContainerSpec{ + { + ImageName: "gvenzl/oracle-free:23.5-slim", Options: oracleOptions(), + }, + } +) + +func oracleOptions() dktest.Options { + cwd, _ := os.Getwd() + mounts := []mount.Mount{ + { + Type: mount.TypeBind, + Source: filepath.Join(cwd, "testdata/init.sql"), + Target: "/docker-entrypoint-initdb.d/init.sql", + }, + } + + return dktest.Options{ + PortRequired: true, + Mounts: mounts, + ReadyFunc: isReady, + ExposedPorts: nat.PortSet{ + nat.Port(fmt.Sprintf("%d/tcp", defaultPort)): {}, + }, + PortBindings: map[nat.Port][]nat.PortBinding{ + nat.Port(fmt.Sprintf("%d/tcp", defaultPort)): { + nat.PortBinding{ + HostIP: "0.0.0.0", + HostPort: "0/tcp", + }, + }, + }, + Env: map[string]string{ + "ORACLE_PASSWORD": defaultPass, + }, + } +} + +func oracleConnectionString(host, port string) string { + return fmt.Sprintf("oracle://%s:%s@%s:%s/FREEPDB1", userdba, userdbaPass, host, port) +} + +func isReady(ctx context.Context, c dktest.ContainerInfo) bool { + ip, port, err := c.Port(defaultPort) + if err != nil { + return false + } + + db, err := sql.Open("oracle", oracleConnectionString(ip, port)) + if err != nil { + return false + } + + defer func() { + if err := db.Close(); err != nil { + log.Println("close error:", err) + } + }() + + if err = db.PingContext(ctx); err != nil { + switch err { + case sqldriver.ErrBadConn, io.EOF: + return false + default: + fmt.Println(err) + } + return false + } + return true +} + +type oracleSuite struct { + dsn string + suite.Suite +} + +// In order for 'go test' to run this suite, we need to create +// a normal test function and pass our suite to suite.Run +func TestOracleTestSuite(t *testing.T) { + if dsn := os.Getenv("ORACLE_DSN"); dsn != "" { + s := oracleSuite{dsn: dsn} + suite.Run(t, &s) + return + } + + dktesting.ParallelTest(t, specs, func(t *testing.T, c dktest.ContainerInfo) { + ip, port, err := c.Port(defaultPort) + if err != nil { + t.Fatal(err) + } + + dsn := oracleConnectionString(ip, port) + s := oracleSuite{dsn: dsn} + + suite.Run(t, &s) + }) +} + +func (s *oracleSuite) TestMigrate() { + ora := &Oracle{} + d, err := ora.Open(s.dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations", "", d) + s.Require().Nil(err) + dt.TestMigrate(s.T(), m) +} + +func (s *oracleSuite) TestMultiStmtMigrate() { + ora := &Oracle{} + dsn := fmt.Sprintf("%s?%s=%s&&%s=%s", s.dsn, multiStmtEnableQueryKey, "true", multiStmtSeparatorQueryKey, "---") + d, err := ora.Open(dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + m, err := migrate.NewWithDatabaseInstance("file://./examples/migrations-multistmt", "", d) + s.Require().Nil(err) + dt.TestMigrate(s.T(), m) +} + +func (s *oracleSuite) TestLockWorks() { + ora := &Oracle{} + d, err := ora.Open(s.dsn) + s.Require().Nil(err) + s.Require().NotNil(d) + defer func() { + if err := d.Close(); err != nil { + s.Error(err) + } + }() + + dt.Test(s.T(), d, []byte(`BEGIN DBMS_OUTPUT.PUT_LINE('hello'); END;`)) + + ora = d.(*Oracle) + err = ora.Lock() + s.Require().Nil(err) + + err = ora.Unlock() + s.Require().Nil(err) + + err = ora.Lock() + s.Require().Nil(err) + + err = ora.Unlock() + s.Require().Nil(err) +} + +func TestParseStatements(t *testing.T) { + cases := []struct { + migration string + expectedQueries []string + }{ + {migration: ` +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); + +--- +-- +BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END; + +--- +-- comment +-- +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); +--- +--`, + expectedQueries: []string{ + `CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +)`, + `BEGIN +EXECUTE IMMEDIATE 'DROP TABLE USERS'; +EXCEPTION + WHEN OTHERS THEN + IF SQLCODE != -942 THEN + RAISE; + END IF; +END;`, + `CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +)`, + }}, + {migration: ` +-- comment +CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +); +-- this is comment +--- +ALTER TABLE USERS ADD CITY varchar(100); +`, + expectedQueries: []string{ + `CREATE TABLE USERS ( + USER_ID integer unique, + NAME varchar(40), + EMAIL varchar(40) +)`, + `ALTER TABLE USERS ADD CITY varchar(100)`, + }}, + } + for _, c := range cases { + queries, err := parseMultiStatements(bytes.NewBufferString(c.migration), DefaultMultiStmtSeparator) + require.Nil(t, err) + require.Equal(t, c.expectedQueries, queries) + } +} diff --git a/database/oracle/testdata/init.sql b/database/oracle/testdata/init.sql new file mode 100644 index 000000000..cc2a34550 --- /dev/null +++ b/database/oracle/testdata/init.sql @@ -0,0 +1,6 @@ +alter session set container=FREEPDB1; +create user orcl identified by orcl; +grant dba to orcl; +grant create session to orcl; +grant connect, resource to orcl; +grant all privileges to orcl; \ No newline at end of file diff --git a/go.mod b/go.mod index f16841a7c..e3dd083be 100644 --- a/go.mod +++ b/go.mod @@ -10,10 +10,11 @@ require ( github.com/Azure/go-autorest/autorest/adal v0.9.16 github.com/ClickHouse/clickhouse-go v1.4.3 github.com/aws/aws-sdk-go v1.49.6 - github.com/cenkalti/backoff/v4 v4.1.2 + github.com/cenkalti/backoff/v4 v4.2.1 github.com/cockroachdb/cockroach-go/v2 v2.1.1 github.com/dhui/dktest v0.4.4 github.com/docker/docker v27.2.0+incompatible + github.com/docker/go-connections v0.5.0 github.com/fsouza/fake-gcs-server v1.17.0 github.com/go-sql-driver/mysql v1.5.0 github.com/gobuffalo/here v0.6.0 @@ -32,6 +33,7 @@ require ( github.com/mutecomm/go-sqlcipher/v4 v4.4.0 github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 github.com/neo4j/neo4j-go-driver v1.8.1-0.20200803113522-b626aa943eba + github.com/sijms/go-ora/v2 v2.8.23 github.com/snowflakedb/gosnowflake v1.6.19 github.com/stretchr/testify v1.9.0 github.com/xanzy/go-gitlab v0.15.0 @@ -46,7 +48,6 @@ require ( require ( github.com/distribution/reference v0.6.0 // indirect - github.com/docker/go-connections v0.5.0 // indirect github.com/felixge/httpsnoop v1.0.4 // indirect github.com/go-logr/logr v1.4.2 // indirect github.com/go-logr/stdr v1.2.2 // indirect @@ -66,6 +67,7 @@ require ( go.opentelemetry.io/otel/sdk v1.29.0 // indirect go.opentelemetry.io/otel/trace v1.29.0 // indirect go.opentelemetry.io/proto/otlp v1.3.1 // indirect + gotest.tools/v3 v3.5.1 // indirect ) require ( @@ -147,7 +149,7 @@ require ( github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/kballard/go-shellquote v0.0.0-20180428030007-95032a82bc51 // indirect github.com/klauspost/asmfmt v1.3.2 // indirect - github.com/klauspost/compress v1.15.11 // indirect + github.com/klauspost/compress v1.17.4 // indirect github.com/klauspost/cpuid/v2 v2.0.9 // indirect github.com/mattn/go-colorable v0.1.6 // indirect github.com/mattn/go-isatty v0.0.16 // indirect diff --git a/go.sum b/go.sum index 3796f207d..1ec7c9eba 100644 --- a/go.sum +++ b/go.sum @@ -121,8 +121,8 @@ github.com/bkaradzic/go-lz4 v1.0.0 h1:RXc4wYsyz985CkXXeX04y4VnZFGG8Rd43pRaHsOXAK github.com/bkaradzic/go-lz4 v1.0.0/go.mod h1:0YdlkowM3VswSROI7qDxhRvJ3sLhlFrRRwjwegp5jy4= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869 h1:DDGfHa7BWjL4YnC6+E63dPcxHo2sUxDIu8g3QgEJdRY= github.com/bmizerany/assert v0.0.0-20160611221934-b7ed37b82869/go.mod h1:Ekp36dRnpXw/yCqJaO+ZrUyxD+3VXMFFr56k5XYrpB4= -github.com/cenkalti/backoff/v4 v4.1.2 h1:6Yo7N8UP2K6LWZnW94DLVSSrbobcWdVzAYOisuDPIFo= -github.com/cenkalti/backoff/v4 v4.1.2/go.mod h1:scbssz8iZGpm3xbr14ovlUdkxfGXNInqkPWOWmG2CLw= +github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= +github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= github.com/census-instrumentation/opencensus-proto v0.4.1 h1:iKLQ0xPNFxR/2hzXZMrBo8f1j86j5WHzznCCQxV/b8g= github.com/census-instrumentation/opencensus-proto v0.4.1/go.mod h1:4T9NM4+4Vw91VeyqjLS6ao50K5bOcLKN6Q42XnYaRYw= @@ -421,8 +421,8 @@ github.com/kisielk/gotool v1.0.0/go.mod h1:XhKaO+MFFWcvkIS/tQcRk01m1F5IRFswLeQ+o github.com/klauspost/asmfmt v1.3.2 h1:4Ri7ox3EwapiOjCki+hw14RyKk201CN4rzyCJRFLpK4= github.com/klauspost/asmfmt v1.3.2/go.mod h1:AG8TuvYojzulgDAMCnYn50l/5QV3Bs/tp6j0HLHbNSE= github.com/klauspost/compress v1.13.6/go.mod h1:/3/Vjq9QcHkK5uEr5lBEmyoZ1iFhe47etQ6QUkpK6sk= -github.com/klauspost/compress v1.15.11 h1:Lcadnb3RKGin4FYM/orgq0qde+nc15E5Cbqg4B9Sx9c= -github.com/klauspost/compress v1.15.11/go.mod h1:QPwzmACJjUTFsnSHH934V6woptycfrDDJnH7hvFVbGM= +github.com/klauspost/compress v1.17.4 h1:Ej5ixsIri7BrIjBkRZLTo6ghwrEtHFk7ijlczPW4fZ4= +github.com/klauspost/compress v1.17.4/go.mod h1:/dCuZOvVtNoHsyb+cuJD3itjs3NbnF6KH9zAO4BDxPM= github.com/klauspost/cpuid/v2 v2.0.9 h1:lgaqFMSdTdQYdZ04uHyN2d/eKdOMyi2YLSvlQIBFYa4= github.com/klauspost/cpuid/v2 v2.0.9/go.mod h1:FInQzS24/EEf25PyTYn52gqo7WaD8xa0213Md/qVLRg= github.com/konsorten/go-windows-terminal-sequences v1.0.1/go.mod h1:T0+1ngSBFLxvqU3pZ+m/2kptfBszLMUkC4ZK/EgS/cQ= @@ -543,6 +543,8 @@ github.com/shopspring/decimal v0.0.0-20180709203117-cd690d0c9e24/go.mod h1:M+9Nz github.com/shopspring/decimal v0.0.0-20200227202807-02e2044944cc/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= github.com/shopspring/decimal v1.2.0 h1:abSATXmQEYyShuxI4/vyW3tV1MrKAJzCZ/0zLUXYbsQ= github.com/shopspring/decimal v1.2.0/go.mod h1:DKyhrW/HYNuLGql+MJL6WCR6knT2jwCFRcu2hWCYk4o= +github.com/sijms/go-ora/v2 v2.8.23 h1:9k4VOty9Nv/Uy8aUqqO90DdRY5pDjKb+QnQ6uimZLiM= +github.com/sijms/go-ora/v2 v2.8.23/go.mod h1:QgFInVi3ZWyqAiJwzBQA+nbKYKH77tdp1PYoCqhR2dU= github.com/sirupsen/logrus v1.4.1/go.mod h1:ni0Sbl8bgC9z8RoU9G6nDWqqs/fq4eDPysMBDgk/93Q= github.com/sirupsen/logrus v1.4.2/go.mod h1:tLMulIdttU9McNUspp0xgXVQah82FyeX6MwdIuYE2rE= github.com/sirupsen/logrus v1.9.3 h1:dueUQJ1C2q9oE3F7wvmSGAaVtTmUizReu6fjN8uqzbQ= @@ -927,8 +929,8 @@ gopkg.in/yaml.v3 v3.0.1/go.mod h1:K4uyk7z7BCEPqu6E+C64Yfv1cQ7kz7rIZviUmN+EgEM= gorm.io/driver/postgres v1.0.8/go.mod h1:4eOzrI1MUfm6ObJU/UcmbXyiHSs8jSwH95G5P5dxcAg= gorm.io/gorm v1.20.12/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= gorm.io/gorm v1.21.4/go.mod h1:0HFTzE/SqkGTzK6TlDPPQbAYCluiVvhzoA1+aVyzenw= -gotest.tools/v3 v3.1.0 h1:rVV8Tcg/8jHUkPUorwjaMTtemIMVXfIPKiOqnhEhakk= -gotest.tools/v3 v3.1.0/go.mod h1:fHy7eyTmJFO5bQbUsEGQ1v4m2J3Jz9eWL54TP2/ZuYQ= +gotest.tools/v3 v3.5.1 h1:EENdUnS3pdur5nybKYIh2Vfgc8IUNBjxDPSjtiJcOzU= +gotest.tools/v3 v3.5.1/go.mod h1:isy3WKz7GK6uNw/sbHzfKBLvlvXwUyV06n6brMxxopU= honnef.co/go/tools v0.0.0-20190102054323-c2f93a96b099/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190106161140-3f1c8253044a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= honnef.co/go/tools v0.0.0-20190418001031-e561f6794a2a/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/internal/cli/build_oracle.go b/internal/cli/build_oracle.go new file mode 100644 index 000000000..c8737a0c1 --- /dev/null +++ b/internal/cli/build_oracle.go @@ -0,0 +1,8 @@ +//go:build oracle +// +build oracle + +package cli + +import ( + _ "github.com/golang-migrate/migrate/v4/database/oracle" +)