From bb8e2924fef6ed2a72ad947a48151bf638f30826 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Mon, 6 May 2024 17:57:05 +0300 Subject: [PATCH 01/17] where-clause --- go/base/context.go | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/go/base/context.go b/go/base/context.go index 41baae696..623282923 100644 --- a/go/base/context.go +++ b/go/base/context.go @@ -123,6 +123,7 @@ type MigrationContext struct { MaxLagMillisecondsThrottleThreshold int64 throttleControlReplicaKeys *mysql.InstanceKeyMap ThrottleFlagFile string + copyWhereClause string ThrottleAdditionalFlagFile string throttleQuery string throttleHTTP string @@ -724,6 +725,19 @@ func (this *MigrationContext) GetCriticalLoad() LoadMap { return this.criticalLoad.Duplicate() } +func (this *MigrationContext) GetWhereClause() string { + this.throttleMutex.Lock() + defer this.throttleMutex.Unlock() + + return this.copyWhereClause +} + +func (this *MigrationContext) SetWhereClause(WhereClause string) { + this.throttleMutex.Lock() + defer this.throttleMutex.Unlock() + this.copyWhereClause = WhereClause +} + func (this *MigrationContext) GetNiceRatio() float64 { this.throttleMutex.Lock() defer this.throttleMutex.Unlock() From 2f4f5550fc9e18b79a4efef0b5caa5ae55297a04 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 09:39:13 +0300 Subject: [PATCH 02/17] where-clause --- .github/CODEOWNERS | 1 - build.sh | 4 ++-- 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS index 95302334c..e69de29bb 100644 --- a/.github/CODEOWNERS +++ b/.github/CODEOWNERS @@ -1 +0,0 @@ -* @rashiq @meiji163 @timvaillancourt diff --git a/build.sh b/build.sh index 64d5b1584..c932624d0 100755 --- a/build.sh +++ b/build.sh @@ -2,8 +2,8 @@ # # -RELEASE_VERSION= -buildpath= +RELEASE_VERSION=2 +buildpath=/tmp function setuptree() { b=$( mktemp -d $buildpath/gh-ostXXXXXX ) || return 1 From 1962c8a97f5e841a926a66c31a8d68b09ccc00dd Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 09:51:40 +0300 Subject: [PATCH 03/17] adding code to main --- go/cmd/gh-ost/main.go | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index 39c815bc8..bd8513960 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -105,17 +105,18 @@ func main() { flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer intervals between failed cut-over attempts. Wait intervals obey a maximum configurable with 'exponential-backoff-max-interval').") exponentialBackoffMaxInterval := flag.Int64("exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between attempts when performing various operations with exponential backoff.") - chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 10-100,000)") + chunkSize := flag.Int64("chunk-size", 100, "amount of rows to handle in each iteration (allowed range: 10-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout)") - niceRatio := flag.Float64("nice-ratio", 0, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after") + niceRatio := flag.Float64("nice-ratio", 1, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after") maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation") replicationLagQuery := flag.String("replication-lag-query", "", "Deprecated. gh-ost uses an internal, subsecond resolution query") throttleControlReplicas := flag.String("throttle-control-replicas", "", "List of replicas on which to check for lag; comma delimited. Example: myhost1.com:3306,myhost2.com,myhost3.com:3307") throttleQuery := flag.String("throttle-query", "", "when given, issued (every second) to check if operation should throttle. Expecting to return zero for no-throttle, >0 for throttle. Query is issued on the migrated server. Make sure this query is lightweight") throttleHTTP := flag.String("throttle-http", "", "when given, gh-ost checks given URL via HEAD request; any response code other than 200 (OK) causes throttling; make sure it has low latency response") + copyWhereClause := flag.String("where-clause", "1=1", "added where clause for the insert query, filtering table rows") flag.Int64Var(&migrationContext.ThrottleHTTPIntervalMillis, "throttle-http-interval-millis", 100, "Number of milliseconds to wait before triggering another HTTP throttle check") flag.Int64Var(&migrationContext.ThrottleHTTPTimeoutMillis, "throttle-http-timeout-millis", 1000, "Number of milliseconds to use as an HTTP throttle check timeout") ignoreHTTPErrors := flag.Bool("ignore-http-errors", false, "ignore HTTP connection errors during throttle check") @@ -304,6 +305,7 @@ func main() { migrationContext.SetThrottleHTTP(*throttleHTTP) migrationContext.SetIgnoreHTTPErrors(*ignoreHTTPErrors) migrationContext.SetDefaultNumRetries(*defaultRetries) + migrationContext.SetWhereClause(*copyWhereClause) migrationContext.ApplyCredentials() if err := migrationContext.SetupTLS(); err != nil { migrationContext.Log.Fatale(err) From 825341d850464cce60558ca8b42814528f4abc13 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 11:01:03 +0300 Subject: [PATCH 04/17] adding code to applier.go --- go/logic/applier.go | 23 ++++++++++++++++++++--- 1 file changed, 20 insertions(+), 3 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 9b190919f..120a1b28b 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -11,6 +11,7 @@ import ( "strings" "sync/atomic" "time" + "context" "github.com/github/gh-ost/go/base" "github.com/github/gh-ost/go/binlog" @@ -634,7 +635,18 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } sqlResult, err := func() (gosql.Result, error) { - tx, err := this.db.Begin() + /*tx, err := this.db.Begin()*/ + var conn *gosql.Conn + conn, err = this.db.Conn(context.Background()) + if (conn == nil || err != nil) { + fmt.Sprintf("failed to get connection") + return nil, err + } + if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=0"); err != nil { + fmt.Sprintf("failed to disable binary logs") + return nil, err + } + tx, err := conn.BeginTx(context.Background(), nil) if err != nil { return nil, err } @@ -643,16 +655,21 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected sessionQuery := fmt.Sprintf(`SET SESSION time_zone = '%s'`, this.migrationContext.ApplierTimeZone) sessionQuery = fmt.Sprintf("%s, %s", sessionQuery, this.generateSqlModeQuery()) - if _, err := tx.Exec(sessionQuery); err != nil { + if _, err := tx.ExecContext(context.Background(), sessionQuery); err != nil { return nil, err } - result, err := tx.Exec(query, explodedArgs...) + result, err := tx.ExecContext(context.Background(), query, explodedArgs...) if err != nil { return nil, err } if err := tx.Commit(); err != nil { return nil, err } + if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=1"); err != nil { + fmt.Sprintf("failed to enable binary logs") + return nil, err + } + conn.Close() return result, nil }() From 104347ffebdc0038e11946b1c6358291ac9cf4ae Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 11:40:22 +0300 Subject: [PATCH 05/17] adding code to build.go --- go/cmd/gh-ost/main.go | 4 ++-- go/logic/applier.go | 1 + go/sql/builder.go | 10 +++++----- 3 files changed, 8 insertions(+), 7 deletions(-) diff --git a/go/cmd/gh-ost/main.go b/go/cmd/gh-ost/main.go index bd8513960..696bc089a 100644 --- a/go/cmd/gh-ost/main.go +++ b/go/cmd/gh-ost/main.go @@ -105,11 +105,11 @@ func main() { flag.BoolVar(&migrationContext.AssumeRBR, "assume-rbr", false, "set to 'true' when you know for certain your server uses 'ROW' binlog_format. gh-ost is unable to tell, event after reading binlog_format, whether the replication process does indeed use 'ROW', and restarts replication to be certain RBR setting is applied. Such operation requires SUPER privileges which you might not have. Setting this flag avoids restarting replication and you can proceed to use gh-ost without SUPER privileges") flag.BoolVar(&migrationContext.CutOverExponentialBackoff, "cut-over-exponential-backoff", false, "Wait exponentially longer intervals between failed cut-over attempts. Wait intervals obey a maximum configurable with 'exponential-backoff-max-interval').") exponentialBackoffMaxInterval := flag.Int64("exponential-backoff-max-interval", 64, "Maximum number of seconds to wait between attempts when performing various operations with exponential backoff.") - chunkSize := flag.Int64("chunk-size", 100, "amount of rows to handle in each iteration (allowed range: 10-100,000)") + chunkSize := flag.Int64("chunk-size", 1000, "amount of rows to handle in each iteration (allowed range: 10-100,000)") dmlBatchSize := flag.Int64("dml-batch-size", 10, "batch size for DML events to apply in a single transaction (range 1-100)") defaultRetries := flag.Int64("default-retries", 60, "Default number of retries for various operations before panicking") cutOverLockTimeoutSeconds := flag.Int64("cut-over-lock-timeout-seconds", 3, "Max number of seconds to hold locks on tables while attempting to cut-over (retry attempted when lock exceeds timeout)") - niceRatio := flag.Float64("nice-ratio", 1, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after") + niceRatio := flag.Float64("nice-ratio", 0, "force being 'nice', imply sleep time per chunk time; range: [0.0..100.0]. Example values: 0 is aggressive. 1: for every 1ms spent copying rows, sleep additional 1ms (effectively doubling runtime); 0.7: for every 10ms spend in a rowcopy chunk, spend 7ms sleeping immediately after") maxLagMillis := flag.Int64("max-lag-millis", 1500, "replication lag at which to throttle operation") replicationLagQuery := flag.String("replication-lag-query", "", "Deprecated. gh-ost uses an internal, subsecond resolution query") diff --git a/go/logic/applier.go b/go/logic/applier.go index 120a1b28b..97aa9a7df 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -629,6 +629,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected this.migrationContext.MigrationIterationRangeMaxValues.AbstractValues(), this.migrationContext.GetIteration() == 0, this.migrationContext.IsTransactionalTable(), + this.migrationContext.GetWhereClause(), ) if err != nil { return chunkSize, rowsAffected, duration, err diff --git a/go/sql/builder.go b/go/sql/builder.go index 7be428f93..a011b3e4e 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -182,7 +182,7 @@ func BuildRangePreparedComparison(columns *ColumnList, args []interface{}, compa return BuildRangeComparison(columns.Names(), values, args, comparisonSign) } -func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartValues, rangeEndValues []string, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool) (result string, explodedArgs []interface{}, err error) { +func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartValues, rangeEndValues []string, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool, whereClause string) (result string, explodedArgs []interface{}, err error) { if len(sharedColumns) == 0 { return "", explodedArgs, fmt.Errorf("Got 0 shared columns in BuildRangeInsertQuery") } @@ -232,19 +232,19 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin %s.%s force index (%s) where - (%s and %s) + ((%s and %s) and (%s)) %s )`, databaseName, originalTableName, databaseName, ghostTableName, mappedSharedColumnsListing, sharedColumnsListing, databaseName, originalTableName, uniqueKey, - rangeStartComparison, rangeEndComparison, transactionalClause) + rangeStartComparison, rangeEndComparison, whereClause, transactionalClause) return result, explodedArgs, nil } -func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool) (result string, explodedArgs []interface{}, err error) { +func BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName string, sharedColumns []string, mappedSharedColumns []string, uniqueKey string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, includeRangeStartValues bool, transactionalTable bool, whereClause string) (result string, explodedArgs []interface{}, err error) { rangeStartValues := buildColumnsPreparedValues(uniqueKeyColumns) rangeEndValues := buildColumnsPreparedValues(uniqueKeyColumns) - return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable) + return BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, includeRangeStartValues, transactionalTable, whereClause) } func BuildUniqueKeyRangeEndPreparedQueryViaOffset(databaseName, tableName string, uniqueKeyColumns *ColumnList, rangeStartArgs, rangeEndArgs []interface{}, chunkSize int64, includeRangeStartValues bool, hint string) (result string, explodedArgs []interface{}, err error) { From 1378868b9da11648e5e6cf6b571d1c4333329476 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 11:52:35 +0300 Subject: [PATCH 06/17] removing sql_bin_log=0 --- go/logic/applier.go | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 97aa9a7df..e7ce10f47 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -643,10 +643,10 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected fmt.Sprintf("failed to get connection") return nil, err } - if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=0"); err != nil { - fmt.Sprintf("failed to disable binary logs") - return nil, err - } +// if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=0"); err != nil { +// fmt.Sprintf("failed to disable binary logs") +// return nil, err +// } tx, err := conn.BeginTx(context.Background(), nil) if err != nil { return nil, err @@ -666,10 +666,10 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected if err := tx.Commit(); err != nil { return nil, err } - if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=1"); err != nil { - fmt.Sprintf("failed to enable binary logs") - return nil, err - } +// if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=1"); err != nil { +// fmt.Sprintf("failed to enable binary logs") +// return nil, err +// } conn.Close() return result, nil }() From 49d1f897c352d443834566f60de856c7cdaaeb5f Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 11:55:28 +0300 Subject: [PATCH 07/17] removing sql_bin_log=0 --- go/sql/builder.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index a011b3e4e..37b0f2699 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -232,7 +232,7 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin %s.%s force index (%s) where - ((%s and %s) and (%s)) + (%s and %s and %s) %s )`, databaseName, originalTableName, databaseName, ghostTableName, mappedSharedColumnsListing, From 63ca4cd4f0094ef7c571e275849d97bfe61d0820 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 12:14:08 +0300 Subject: [PATCH 08/17] remove code_owners file --- .github/CODEOWNERS | 0 1 file changed, 0 insertions(+), 0 deletions(-) delete mode 100644 .github/CODEOWNERS diff --git a/.github/CODEOWNERS b/.github/CODEOWNERS deleted file mode 100644 index e69de29bb..000000000 From 2bb7cb9a9c9f75ac577911460a078a1cbb4a9a9d Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 13:44:12 +0300 Subject: [PATCH 09/17] remove PR issued --- .github/ISSUE_TEMPLATE.md | 13 ------------- .github/PULL_REQUEST_TEMPLATE.md | 19 ------------------- 2 files changed, 32 deletions(-) delete mode 100644 .github/ISSUE_TEMPLATE.md delete mode 100644 .github/PULL_REQUEST_TEMPLATE.md diff --git a/.github/ISSUE_TEMPLATE.md b/.github/ISSUE_TEMPLATE.md deleted file mode 100644 index 820d4ef96..000000000 --- a/.github/ISSUE_TEMPLATE.md +++ /dev/null @@ -1,13 +0,0 @@ -> This is the place to report a bug, ask a question, or suggest an enhancement. - -> This is also the place to make a discussion before creating a PR. - -> If this is a bug report, please provide a test case (e.g., your table definition and gh-ost command) and the error output. - -> Please use markdown to format code or SQL: https://guides.github.com/features/mastering-markdown/ - -> Please label the issue on the right (bug, enhancement, question, etc.). - -> And please understand if this issue is not addressed immediately or in a timeframe you were expecting. - -> Thank you! diff --git a/.github/PULL_REQUEST_TEMPLATE.md b/.github/PULL_REQUEST_TEMPLATE.md deleted file mode 100644 index c0c246463..000000000 --- a/.github/PULL_REQUEST_TEMPLATE.md +++ /dev/null @@ -1,19 +0,0 @@ -## A Pull Request should be associated with an Issue. - -> We wish to have discussions in Issues. A single issue may be targeted by multiple PRs. -> If you're offering a new feature or fixing anything, we'd like to know beforehand in Issues, -> and potentially we'll be able to point development in a particular direction. - -Related issue: https://github.com/github/gh-ost/issues/0123456789 - -> Further notes in https://github.com/github/gh-ost/blob/master/.github/CONTRIBUTING.md -> Thank you! We are open to PRs, but please understand if for technical reasons we are unable to accept each and any PR - -### Description - -This PR [briefly explain what it does] - -> In case this PR introduced Go code changes: - -- [ ] contributed code is using same conventions as original code -- [ ] `script/cibuild` returns with no formatting errors, build errors or unit test errors. From c99aaa4ee0853b642bfc043808cba4e71f429964 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 14:27:19 +0300 Subject: [PATCH 10/17] remove comments --- go/logic/applier.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index e7ce10f47..e8ffed670 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -636,17 +636,12 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected } sqlResult, err := func() (gosql.Result, error) { - /*tx, err := this.db.Begin()*/ var conn *gosql.Conn conn, err = this.db.Conn(context.Background()) if (conn == nil || err != nil) { fmt.Sprintf("failed to get connection") return nil, err } -// if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=0"); err != nil { -// fmt.Sprintf("failed to disable binary logs") -// return nil, err -// } tx, err := conn.BeginTx(context.Background(), nil) if err != nil { return nil, err @@ -666,10 +661,6 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected if err := tx.Commit(); err != nil { return nil, err } -// if _, err := conn.ExecContext(context.Background(), "SET @@SESSION.sql_log_bin=1"); err != nil { -// fmt.Sprintf("failed to enable binary logs") -// return nil, err -// } conn.Close() return result, nil }() From d66cc5566e7ca694644b2b4328d6714498eb1053 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 14:49:54 +0300 Subject: [PATCH 11/17] fix tests --- go/sql/builder_test.go | 23 +++++++++++++---------- 1 file changed, 13 insertions(+), 10 deletions(-) diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 574e8bb1b..9c87ad43c 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -163,6 +163,7 @@ func TestBuildRangeInsertQuery(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" ghostTableName := "ghost" + whereClause := "1=1" sharedColumns := []string{"id", "name", "position"} { uniqueKey := "PRIMARY" @@ -172,7 +173,7 @@ func TestBuildRangeInsertQuery(t *testing.T) { rangeStartArgs := []interface{}{3} rangeEndArgs := []interface{}{103} - query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false) + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) test.S(t).ExpectNil(err) expected := ` insert /* gh-ost mydb.tbl */ ignore @@ -186,7 +187,7 @@ func TestBuildRangeInsertQuery(t *testing.T) { force index (PRIMARY) where (((id > @v1s) or ((id = @v1s))) - and ((id < @v1e) or ((id = @v1e)))) + and ((id < @v1e) or ((id = @v1e))) and 1=1) )` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 103, 103})) @@ -199,7 +200,7 @@ func TestBuildRangeInsertQuery(t *testing.T) { rangeStartArgs := []interface{}{3, 17} rangeEndArgs := []interface{}{103, 117} - query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false) + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) test.S(t).ExpectNil(err) expected := ` insert /* gh-ost mydb.tbl */ ignore @@ -219,7 +220,7 @@ func TestBuildRangeInsertQuery(t *testing.T) { and ((name < @v1e) or (((name = @v1e)) AND (position < @v2e)) - or ((name = @v1e) and (position = @v2e)))) + or ((name = @v1e) and (position = @v2e))) and 1=1) )` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 17, 3, 17, 103, 103, 117, 103, 117})) @@ -230,6 +231,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" ghostTableName := "ghost" + whereClause := "1=1" sharedColumns := []string{"id", "name", "position"} mappedSharedColumns := []string{"id", "name", "location"} { @@ -240,7 +242,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) { rangeStartArgs := []interface{}{3} rangeEndArgs := []interface{}{103} - query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false) + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) test.S(t).ExpectNil(err) expected := ` insert /* gh-ost mydb.tbl */ ignore @@ -255,7 +257,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) { where (((id > @v1s) or ((id = @v1s))) and - ((id < @v1e) or ((id = @v1e)))) + ((id < @v1e) or ((id = @v1e))) and 1=1) )` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 103, 103})) @@ -268,7 +270,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) { rangeStartArgs := []interface{}{3, 17} rangeEndArgs := []interface{}{103, 117} - query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false) + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, mappedSharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) test.S(t).ExpectNil(err) expected := ` insert /* gh-ost mydb.tbl */ ignore @@ -284,7 +286,7 @@ func TestBuildRangeInsertQueryRenameMap(t *testing.T) { (((name > @v1s) or (((name = @v1s)) AND (position > @v2s)) or ((name = @v1s) and (position = @v2s))) and ((name < @v1e) or (((name = @v1e)) AND (position < @v2e)) - or ((name = @v1e) and (position = @v2e)))) + or ((name = @v1e) and (position = @v2e))) and 1=1) )` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 17, 3, 17, 103, 103, 117, 103, 117})) @@ -295,6 +297,7 @@ func TestBuildRangeInsertPreparedQuery(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" ghostTableName := "ghost" + whereClause := "1=1" sharedColumns := []string{"id", "name", "position"} { uniqueKey := "name_position_uidx" @@ -302,7 +305,7 @@ func TestBuildRangeInsertPreparedQuery(t *testing.T) { rangeStartArgs := []interface{}{3, 17} rangeEndArgs := []interface{}{103, 117} - query, explodedArgs, err := BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, true, true) + query, explodedArgs, err := BuildRangeInsertPreparedQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartArgs, rangeEndArgs, true, true, whereClause) test.S(t).ExpectNil(err) expected := ` insert /* gh-ost mydb.tbl */ ignore @@ -314,7 +317,7 @@ func TestBuildRangeInsertPreparedQuery(t *testing.T) { from mydb.tbl force index (name_position_uidx) - where (((name > ?) or (((name = ?)) AND (position > ?)) or ((name = ?) and (position = ?))) and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?)))) + where (((name > ?) or (((name = ?)) AND (position > ?)) or ((name = ?) and (position = ?))) and ((name < ?) or (((name = ?)) AND (position < ?)) or ((name = ?) and (position = ?))) and 1=1) lock in share mode )` test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) From 8de94e32a30a568426c3e0eaafcdb4422bec5842 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Tue, 7 May 2024 15:06:33 +0300 Subject: [PATCH 12/17] test --- go/sql/builder_test.go | 92 ++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 92 insertions(+) diff --git a/go/sql/builder_test.go b/go/sql/builder_test.go index 9c87ad43c..3766146b5 100644 --- a/go/sql/builder_test.go +++ b/go/sql/builder_test.go @@ -227,6 +227,98 @@ func TestBuildRangeInsertQuery(t *testing.T) { } } + +func TestBuildRangeInsertQueryWhereClauseFiltering(t *testing.T) { + databaseName := "mydb" + originalTableName := "tbl" + ghostTableName := "ghost" + sharedColumns := []string{"id", "name", "position"} + { + uniqueKey := "PRIMARY" + uniqueKeyColumns := NewColumnList([]string{"id"}) + rangeStartValues := []string{"@v1s"} + rangeEndValues := []string{"@v1e"} + rangeStartArgs := []interface{}{3} + rangeEndArgs := []interface{}{103} + whereClause := "id = 1" + + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) + test.S(t).ExpectNil(err) + expected := ` + insert /* gh-ost mydb.tbl */ ignore + into + mydb.ghost + (id, name, position) + ( + select id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > @v1s) or ((id = @v1s))) + and ((id < @v1e) or ((id = @v1e))) and id = 1) + )` + test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) + test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 103, 103})) + } + { + uniqueKey := "PRIMARY" + uniqueKeyColumns := NewColumnList([]string{"id"}) + rangeStartValues := []string{"@v1s"} + rangeEndValues := []string{"@v1e"} + rangeStartArgs := []interface{}{3} + rangeEndArgs := []interface{}{103} + whereClause := "id not in (1,2,3)" + + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) + test.S(t).ExpectNil(err) + expected := ` + insert /* gh-ost mydb.tbl */ ignore + into + mydb.ghost + (id, name, position) + ( + select id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > @v1s) or ((id = @v1s))) + and ((id < @v1e) or ((id = @v1e))) and id not in (1,2,3)) + )` + test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) + test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 103, 103})) + } + { + uniqueKey := "PRIMARY" + uniqueKeyColumns := NewColumnList([]string{"id"}) + rangeStartValues := []string{"@v1s"} + rangeEndValues := []string{"@v1e"} + rangeStartArgs := []interface{}{3} + rangeEndArgs := []interface{}{103} + whereClause := "id in (select id from ids)" + + query, explodedArgs, err := BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName, sharedColumns, sharedColumns, uniqueKey, uniqueKeyColumns, rangeStartValues, rangeEndValues, rangeStartArgs, rangeEndArgs, true, false, whereClause) + test.S(t).ExpectNil(err) + expected := ` + insert /* gh-ost mydb.tbl */ ignore + into + mydb.ghost + (id, name, position) + ( + select id, name, position + from + mydb.tbl + force index (PRIMARY) + where + (((id > @v1s) or ((id = @v1s))) + and ((id < @v1e) or ((id = @v1e))) and id in (select id from ids)) + )` + test.S(t).ExpectEquals(normalizeQuery(query), normalizeQuery(expected)) + test.S(t).ExpectTrue(reflect.DeepEqual(explodedArgs, []interface{}{3, 3, 103, 103})) + } +} + func TestBuildRangeInsertQueryRenameMap(t *testing.T) { databaseName := "mydb" originalTableName := "tbl" From 045a657ef872cb964761adf8382c23909d8015f6 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Sun, 12 May 2024 11:14:18 +0300 Subject: [PATCH 13/17] adding debug and status --- go/logic/migrator.go | 3 ++- go/sql/builder.go | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/go/logic/migrator.go b/go/logic/migrator.go index fed7c944b..8c18263f3 100644 --- a/go/logic/migrator.go +++ b/go/logic/migrator.go @@ -838,13 +838,14 @@ func (this *Migrator) printMigrationStatusHint(writers ...io.Writer) { ) maxLoad := this.migrationContext.GetMaxLoad() criticalLoad := this.migrationContext.GetCriticalLoad() - fmt.Fprintf(w, "# chunk-size: %+v; max-lag-millis: %+vms; dml-batch-size: %+v; max-load: %s; critical-load: %s; nice-ratio: %f\n", + fmt.Fprintf(w, "# chunk-size: %+v; max-lag-millis: %+vms; dml-batch-size: %+v; max-load: %s; critical-load: %s; nice-ratio: %f; where-clause: %s\n", atomic.LoadInt64(&this.migrationContext.ChunkSize), atomic.LoadInt64(&this.migrationContext.MaxLagMillisecondsThrottleThreshold), atomic.LoadInt64(&this.migrationContext.DMLBatchSize), maxLoad.String(), criticalLoad.String(), this.migrationContext.GetNiceRatio(), + this.migrationContext.GetWhereClause(), ) if this.migrationContext.ThrottleFlagFile != "" { setIndicator := "" diff --git a/go/sql/builder.go b/go/sql/builder.go index 37b0f2699..879233d4d 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -238,6 +238,7 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin databaseName, originalTableName, databaseName, ghostTableName, mappedSharedColumnsListing, sharedColumnsListing, databaseName, originalTableName, uniqueKey, rangeStartComparison, rangeEndComparison, whereClause, transactionalClause) + fmt.Sprintf("the query: ", result) return result, explodedArgs, nil } From dbdb89f6a73c006ab416674c407182f7f132e27b Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Sun, 12 May 2024 11:14:34 +0300 Subject: [PATCH 14/17] adding where clause to status --- go/sql/builder.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/sql/builder.go b/go/sql/builder.go index 879233d4d..37b0f2699 100644 --- a/go/sql/builder.go +++ b/go/sql/builder.go @@ -238,7 +238,6 @@ func BuildRangeInsertQuery(databaseName, originalTableName, ghostTableName strin databaseName, originalTableName, databaseName, ghostTableName, mappedSharedColumnsListing, sharedColumnsListing, databaseName, originalTableName, uniqueKey, rangeStartComparison, rangeEndComparison, whereClause, transactionalClause) - fmt.Sprintf("the query: ", result) return result, explodedArgs, nil } From a611d80d75c1391305b660a36348c706f4d67d37 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Sun, 12 May 2024 15:03:22 +0300 Subject: [PATCH 15/17] add quer debug --- go/logic/applier.go | 1 + 1 file changed, 1 insertion(+) diff --git a/go/logic/applier.go b/go/logic/applier.go index e8ffed670..7124ba40b 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -631,6 +631,7 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected this.migrationContext.IsTransactionalTable(), this.migrationContext.GetWhereClause(), ) + this.migrationContext.Log.Debugf("query = %s", query) if err != nil { return chunkSize, rowsAffected, duration, err } From 6de0e3ea28dc1a864bf2ede0a30b138e837ab059 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Mon, 13 May 2024 09:12:24 +0300 Subject: [PATCH 16/17] remove debug line --- go/logic/applier.go | 1 - 1 file changed, 1 deletion(-) diff --git a/go/logic/applier.go b/go/logic/applier.go index 7124ba40b..e8ffed670 100644 --- a/go/logic/applier.go +++ b/go/logic/applier.go @@ -631,7 +631,6 @@ func (this *Applier) ApplyIterationInsertQuery() (chunkSize int64, rowsAffected this.migrationContext.IsTransactionalTable(), this.migrationContext.GetWhereClause(), ) - this.migrationContext.Log.Debugf("query = %s", query) if err != nil { return chunkSize, rowsAffected, duration, err } From 98dc355804abd989f3ef5757b6b98398e28c01f2 Mon Sep 17 00:00:00 2001 From: BarShauli555 Date: Sun, 18 Aug 2024 15:26:36 +0300 Subject: [PATCH 17/17] count_rows_queries --- go/logic/inspect.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/logic/inspect.go b/go/logic/inspect.go index 9d414a43e..b5a7c79e9 100644 --- a/go/logic/inspect.go +++ b/go/logic/inspect.go @@ -554,7 +554,7 @@ func (this *Inspector) CountTableRows(ctx context.Context) error { return err } - query := fmt.Sprintf(`select /* gh-ost */ count(*) as count_rows from %s.%s`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) + query := fmt.Sprintf(`select /* gh-ost */ table_rows as count_rows from information_schema.tables where table_schema="%s" and table_name="%s"`, sql.EscapeName(this.migrationContext.DatabaseName), sql.EscapeName(this.migrationContext.OriginalTableName)) var rowsEstimate int64 if err := conn.QueryRowContext(ctx, query).Scan(&rowsEstimate); err != nil { if errors.Is(err, context.Canceled) || errors.Is(err, context.DeadlineExceeded) {