Skip to content

Commit

Permalink
feat: data compare tidb, close #28
Browse files Browse the repository at this point in the history
  • Loading branch information
@wentaojin
wentaojin committed Jul 19, 2024
1 parent 8389c1e commit 556e560
Show file tree
Hide file tree
Showing 75 changed files with 3,762 additions and 2,673 deletions.
1 change: 0 additions & 1 deletion Makefile
View file
Original file line number Diff line number Diff line change
Expand Up @@ -59,7 +59,6 @@ GOENV := GO111MODULE=on CGO_ENABLED=1 GOOS=$(GOOS) GOARCH=$(GOARCH)
GO := $(GOENV) go
GOBUILD := $(GO) build
GORUN := $(GO) run
SHELL := /usr/bin/env bash

COMMIT := $(shell git describe --always --no-match --tags --dirty="-dev")
BUILDTS := $(shell date -u '+%Y-%m-%d %H:%M:%S')
Expand Down
2 changes: 2 additions & 0 deletions component/cli/datasource/datasource.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,8 @@ type Datasource struct {
ConnectStatus string `toml:"connect-status" json:"connectStatus"`
ServiceName string `toml:"service-name" json:"serviceName"`
PdbName string `toml:"pdb-name" json:"pdbName"`
SessionParams string `toml:"session-params" json:"sessionParams"`
DbName string `toml:"db-name" json:"dbName"`
Comment string `toml:"comment" json:"comment"`
}

Expand Down
35 changes: 19 additions & 16 deletions component/cli/migrate/verify.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -40,25 +40,28 @@ type CompareConfig struct {
}

type DataCompareRule struct {
TableNameS string `toml:"table-name-s" json:"tableNameS"`
CompareField string `toml:"compare-field" json:"compareField"`
CompareRange string `toml:"compare-range" json:"compareRange"`
IgnoreFields []string `toml:"ignore-fields" json:"ignoreFields"`
TableNameS string `toml:"table-name-s" json:"tableNameS"`
CompareConditionField string `toml:"compare-condition-field" json:"compareConditionField"`
CompareConditionRange string `toml:"compare-condition-range" json:"compareConditionRange"`
IgnoreSelectFields []string `toml:"ignore-select-fields" json:"ignoreSelectFields"`
IgnoreConditionFields []string `toml:"ignore-condition-fields" json:"ignoreConditionFields"`
}

type DataCompareParam struct {
TableThread uint64 `toml:"table-thread" json:"tableThread"`
BatchSize uint64 `toml:"batch-size" json:"batchSize"`
SqlThread uint64 `toml:"sql-thread" json:"sqlThread"`
WriteThread uint64 `toml:"write-thread" json:"writeThread"`
SqlHintS string `toml:"sql-hint-s" json:"sqlHintS"`
SqlHintT string `toml:"sql-hint-t" json:"sqlHintT"`
CallTimeout uint64 `toml:"call-timeout" json:"callTimeout"`
EnableCheckpoint bool `toml:"enable-checkpoint" json:"enableCheckpoint"`
EnableConsistentRead bool `toml:"enable-consistent-read" json:"enableConsistentRead"`
OnlyCompareRow bool `toml:"only-compare-row" json:"onlyCompareRow"`
ConsistentReadPointS string `toml:"consistent-read-point-s" json:"consistentReadPointS"`
ConsistentReadPointT string `toml:"consistent-read-point-t" json:"consistentReadPointT"`
TableThread uint64 `toml:"table-thread" json:"tableThread"`
BatchSize uint64 `toml:"batch-size" json:"batchSize"`
SqlThread uint64 `toml:"sql-thread" json:"sqlThread"`
ChunkSize uint64 `toml:"chunk-size" json:"chunkSize"`
WriteThread uint64 `toml:"write-thread" json:"writeThread"`
SqlHintS string `toml:"sql-hint-s" json:"sqlHintS"`
SqlHintT string `toml:"sql-hint-t" json:"sqlHintT"`
CallTimeout uint64 `toml:"call-timeout" json:"callTimeout"`
EnableCheckpoint bool `toml:"enable-checkpoint" json:"enableCheckpoint"`
EnableConsistentRead bool `toml:"enable-consistent-read" json:"enableConsistentRead"`
OnlyCompareRow bool `toml:"only-compare-row" json:"onlyCompareRow"`
ConsistentReadPointS string `toml:"consistent-read-point-s" json:"consistentReadPointS"`
ConsistentReadPointT string `toml:"consistent-read-point-t" json:"consistentReadPointT"`
IgnoreConditionFields []string `toml:"ignore-condition-fields" json:"ignoreConditionFields"`
}

func (d *CompareConfig) String() string {
Expand Down
73 changes: 41 additions & 32 deletions database/data_compare.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -23,38 +23,41 @@ import (
)

type IDatabaseDataCompare interface {
GetDatabaseTableStatisticsBucket(schemeNameS, tableNameS string) (map[string][]structure.Bucket, error)
GetDatabaseTableStatisticsHistogram(schemeNameS, tableNameS string) (map[string][]structure.Histogram, error)
GetDatabaseTableColumnProperties(schemaNameS, tableNameS, columnNameS string, collationS bool) ([]map[string]string, error)
GetDatabaseTableConstraintIndexColumn(schemaNameS, tableNameS string) (map[string]string, error)
GetDatabaseTableStatisticsBucket(schemeNameS, tableNameS string, consColumns map[string]string) (map[string][]structure.Bucket, error)
GetDatabaseTableStatisticsHistogram(schemeNameS, tableNameS string, consColumns map[string]string) (map[string]structure.Histogram, error)
GetDatabaseTableColumnProperties(schemaNameS, tableNameS string, columnNameSli []string) ([]map[string]string, error)
GetDatabaseTableHighestSelectivityIndex(schemaNameS, tableNameS string, compareCondField string, ignoreCondFields []string) (*structure.HighestBucket, error)
GetDatabaseTableRandomValues(schemaNameS, tableNameS string, columns []string, conditions string, limit int, collations []string) ([][]string, error)
GetDatabaseTableCompareData(querySQL string, callTimeout int, dbCharsetS, dbCharsetT string) ([]string, uint32, map[string]int64, error)
FindDatabaseTableBestColumn(schemaNameS, tableNameS, columnNameS string) ([]string, error)
GetDatabaseTableColumnBucket(schemaNameS, tableNameS string, columnNameS, datatypeS string) ([]string, error)
}

// IDataCompareRuleInitializer used for database table rule initializer
type IDataCompareRuleInitializer interface {
GenSchemaTableCompareMethodRule() string
GenSchemaTableCustomRule() (string, string, error)
GenSchemaTableCustomRule() (string, string, []string, error)
IDatabaseSchemaTableRule
}

type DataCompareAttributesRule struct {
SchemaNameS string `json:"schemaNameS"`
SchemaNameT string `json:"schemaNameT"`
TableNameS string `json:"tableNameS"`
TableTypeS string `json:"tableTypeS"`
TableNameT string `json:"tableNameT"`
ColumnDetailSO string `json:"columnDetailSO"`
ColumnDetailS string `json:"columnDetailS"`
ColumnDetailT string `json:"columnDetailT"`
ColumnDetailTO string `json:"columnDetailTO"`
CompareMethod string `json:"compareMethod"`
ColumnFieldC string `json:"columnFieldC"`
CompareRangeC string `json:"compareRangeC"`
SchemaNameS string `json:"schemaNameS"`
SchemaNameT string `json:"schemaNameT"`
TableNameS string `json:"tableNameS"`
TableTypeS string `json:"tableTypeS"`
TableNameT string `json:"tableNameT"`
ColumnDetailSO string `json:"columnDetailSO"`
ColumnDetailS string `json:"columnDetailS"`
ColumnDetailT string `json:"columnDetailT"`
ColumnDetailTO string `json:"columnDetailTO"`
CompareMethod string `json:"compareMethod"`
ColumnNameRouteRule map[string]string `json:"columnNameRouteRule"` // keep the column name upstream and downstream mapping, a -> b
CompareConditionFieldC string `json:"compareConditionFieldC"`
CompareConditionRangeC string `json:"compareConditionRangeC"`
IgnoreConditionFields []string `json:"ignoreConditionFields"`
}

func IDataCompareAttributesRule(i IDataCompareRuleInitializer) (*DataCompareAttributesRule, error) {
columnFields, compareRange, err := i.GenSchemaTableCustomRule()
columnFields, compareRange, ignoreConditionFields, err := i.GenSchemaTableCustomRule()
if err != nil {
return &DataCompareAttributesRule{}, err
}
Expand All @@ -66,23 +69,29 @@ func IDataCompareAttributesRule(i IDataCompareRuleInitializer) (*DataCompareAttr
if err != nil {
return &DataCompareAttributesRule{}, err
}
sourceColumnSO, sourceColumnS, targetColumnTO, targetColumnT, err := i.GenSchemaTableColumnRule()
sourceColumnSO, sourceColumnS, targetColumnTO, targetColumnT, err := i.GenSchemaTableColumnSelectRule()
if err != nil {
return &DataCompareAttributesRule{}, err
}
columnRouteRule, err := i.GetSchemaTableColumnNameRule()
if err != nil {
return nil, err
}
return &DataCompareAttributesRule{
SchemaNameS: sourceSchema,
SchemaNameT: targetSchema,
TableNameS: sourceTable,
TableNameT: targetTable,
TableTypeS: i.GenSchemaTableTypeRule(),
ColumnDetailSO: sourceColumnSO,
ColumnDetailS: sourceColumnS,
ColumnDetailTO: targetColumnTO,
ColumnDetailT: targetColumnT,
CompareMethod: i.GenSchemaTableCompareMethodRule(),
ColumnFieldC: columnFields,
CompareRangeC: compareRange,
SchemaNameS: sourceSchema,
SchemaNameT: targetSchema,
TableNameS: sourceTable,
TableNameT: targetTable,
TableTypeS: i.GenSchemaTableTypeRule(),
ColumnDetailSO: sourceColumnSO,
ColumnDetailS: sourceColumnS,
ColumnDetailTO: targetColumnTO,
ColumnDetailT: targetColumnT,
CompareMethod: i.GenSchemaTableCompareMethodRule(),
CompareConditionFieldC: columnFields,
CompareConditionRangeC: compareRange,
IgnoreConditionFields: ignoreConditionFields,
ColumnNameRouteRule: columnRouteRule,
}, nil
}

Expand Down
4 changes: 2 additions & 2 deletions database/data_migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -26,7 +26,7 @@ type IDatabaseDataMigrate interface {
GetDatabaseCharset() (string, error)
GetDatabaseConsistentPos() (uint64, error)
GetDatabaseTableType(schemaName string) (map[string]string, error)
GetDatabaseTableColumnInfo(schemaName string, tableName string, collation bool) ([]map[string]string, error)
GetDatabaseTableColumnInfo(schemaName string, tableName string) ([]map[string]string, error)
GetDatabaseTableColumnNameTableDimensions(schemaName, tableName string) ([]string, error)
GetDatabaseTableColumnNameSqlDimensions(sqlStr string) ([]string, map[string]string, map[string]string, error)
GetDatabaseTableRows(schemaName, tableName string) (uint64, error)
Expand Down Expand Up @@ -65,7 +65,7 @@ func IDataMigrateAttributesRule(i IDataMigrateRuleInitializer) (*DataMigrateAttr
if err != nil {
return &DataMigrateAttributesRule{}, err
}
sourceColumnO, sourceColumnS, _, targetColumnT, err := i.GenSchemaTableColumnRule()
sourceColumnO, sourceColumnS, _, targetColumnT, err := i.GenSchemaTableColumnSelectRule()
if err != nil {
return &DataMigrateAttributesRule{}, err
}
Expand Down
3 changes: 2 additions & 1 deletion database/database.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,8 @@ type IDatabaseSchemaTableRule interface {
GenSchemaTableTypeRule() string
GenSchemaNameRule() (string, string, error)
GenSchemaTableNameRule() (string, string, error)
GenSchemaTableColumnRule() (string, string, string, string, error)
GetSchemaTableColumnNameRule() (map[string]string, error)
GenSchemaTableColumnSelectRule() (string, string, string, string, error)
}

func NewDatabase(ctx context.Context, datasource *datasource.Datasource, migrateOracleSchema string) (IDatabase, error) {
Expand Down
Loading

0 comments on commit 556e560

Please sign in to comment.