Skip to content

Commit

Permalink
fixes: ora desc index, close #87
Browse files Browse the repository at this point in the history
  • Loading branch information
@wentaojin
wentaojin committed Dec 27, 2024
1 parent 9c13cce commit 3a0c490
Show file tree
Hide file tree
Showing 4 changed files with 83 additions and 10 deletions.
4 changes: 2 additions & 2 deletions database/processor/struct_migrate_table.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1057,14 +1057,14 @@ func (t *StructMigrateTable) GenTableNormalIndex() ([]string, []string, error) {
brackets := stringutil.StringExtractorWithinBrackets(col)
if len(brackets) == 0 {
// NAME9 PART
if val, ok := t.TableAttributesRule.ColumnNameRule[col]; ok {
if val, ok := t.TableAttributesRule.ColumnNameRule[stringutil.StringReplacer(col, "\"", "")]; ok {
convertTargetRaw, err = stringutil.CharsetConvert([]byte(val), constant.CharsetUTF8MB4, constant.MigrateMySQLCompatibleCharsetStringConvertMapping[stringutil.StringUpper(t.DBCharsetT)])
if err != nil {
return nil, nil, fmt.Errorf("[GenTableNormalIndex] the upstream database schema [%s] table [%s] column [%s] normal_index charset convert [%s] failed, error: %v", t.DatasourceS.SchemaNameS, t.DatasourceS.TableNameS, idxMeta["COLUMN_LIST"], t.DBCharsetT, err)
}
normalIndex[i] = fmt.Sprintf("`%s`", stringutil.BytesToString(convertTargetRaw))
} else {
return nil, nil, fmt.Errorf("[GenTableNormalIndex] the upstream database schema [%v] table [%v] column [%v] isn't exist, please contact author or recheck", t.DatasourceS.SchemaNameS, t.DatasourceS.TableNameS, col)
return nil, nil, fmt.Errorf("[GenTableNormalIndex] the upstream database schema [%v] table [%v] column [%v] isn't existed in [%v], origin column list [%s], please contact author or recheck", t.DatasourceS.SchemaNameS, t.DatasourceS.TableNameS, col, t.TableAttributesRule.ColumnNameRule, columnList)
}
} else {
for k, v := range t.TableAttributesRule.ColumnNameRule {
Expand Down
64 changes: 60 additions & 4 deletions database/processor/struct_migrate_task.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -614,6 +614,14 @@ func (st *StructMigrateTask) processStructMigrate() error {
}

func (st *StructMigrateTask) structMigrateStart(smt *task.StructMigrateTask) error {
logger.Info("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.Uint64("struct_migrate_id", smt.ID))

// if the schema table success, skip
if strings.EqualFold(smt.TaskStatus, constant.TaskDatabaseStatusSuccess) {
logger.Warn("struct migrate task processor",
Expand Down Expand Up @@ -686,14 +694,26 @@ func (st *StructMigrateTask) structMigrateStart(smt *task.StructMigrateTask) err
if err != nil {
return err
}

logger.Info("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("datasource", datasourceS.String()),
zap.String("task_stage", "struct attributes gen"),
zap.String("task_stage", "struct attribute gen"),
zap.String("cost", time.Now().Sub(sourceTime).String()))

logger.Debug("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("datasource", datasourceS.String()),
zap.String("arrtibutes", attrs.String()),
zap.String("task_stage", "struct attribute gen"),
zap.String("cost", time.Now().Sub(sourceTime).String()))

ruleTime := time.Now()
Expand All @@ -717,6 +737,25 @@ func (st *StructMigrateTask) structMigrateStart(smt *task.StructMigrateTask) err
BuildinDefaultValueRules: st.BuildInDefaultValueRules,
}

logger.Info("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("task_stage", "struct rule prepare"),
zap.String("cost", time.Now().Sub(ruleTime).String()))

logger.Debug("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("migrate_rules", dataRule.String()),
zap.String("task_stage", "struct rule prepare"),
zap.String("cost", time.Now().Sub(ruleTime).String()))

rules, err := database.IStructMigrateAttributesRule(dataRule)
if err != nil {
return err
Expand All @@ -727,7 +766,16 @@ func (st *StructMigrateTask) structMigrateStart(smt *task.StructMigrateTask) err
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("datasource", datasourceS.String()),
zap.String("task_stage", "struct rule gen"),
zap.String("cost", time.Now().Sub(ruleTime).String()))

logger.Debug("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("struct_rules", rules.String()),
zap.String("task_stage", "struct rule gen"),
zap.String("cost", time.Now().Sub(ruleTime).String()))

Expand All @@ -745,14 +793,22 @@ func (st *StructMigrateTask) structMigrateStart(smt *task.StructMigrateTask) err
if err != nil {
return err
}

logger.Info("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("datasource", datasourceS.String()),
zap.String("task_stage", "struct table gen"),
zap.String("cost", time.Now().Sub(tableTime).String()))

logger.Debug("struct migrate task processor",
zap.String("task_name", st.Task.TaskName),
zap.String("task_mode", st.Task.TaskMode),
zap.String("task_flow", st.Task.TaskFlow),
zap.String("schema_name_s", st.SchemaNameS),
zap.String("schema_name_t", st.SchemaNameT),
zap.String("struct_table", tableStruct.String()),
zap.String("task_stage", "struct table gen"),
zap.String("cost", time.Now().Sub(tableTime).String()))

Expand Down
17 changes: 17 additions & 0 deletions database/struct_migrate.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,8 @@ limitations under the License.
*/
package database

import "github.com/wentaojin/dbms/utils/stringutil"

// IDatabaseStructMigrate used for database table struct migrate
type IDatabaseStructMigrate interface {
GetDatabaseSchema() ([]string, error)
Expand Down Expand Up @@ -80,6 +82,11 @@ type StructMigrateAttributes struct {
OriginStruct string `json:"originStruct"`
}

func (s *StructMigrateAttributes) String() string {
js, _ := stringutil.MarshalIndentJSON(s)
return js
}

func IStructMigrateAttributes(t IStructMigrateAttributesReader) (*StructMigrateAttributes, error) {
charset, collation, err := t.GetTableCharsetCollation()
if err != nil {
Expand Down Expand Up @@ -174,6 +181,11 @@ type StructMigrateAttributesRule struct {
TableCommentRule string `json:"tableCommentRule"`
}

func (s *StructMigrateAttributesRule) String() string {
js, _ := stringutil.MarshalIndentJSON(s)
return js
}

func IStructMigrateAttributesRule(t IStructMigrateAttributesRuleReader) (*StructMigrateAttributesRule, error) {
schemaNameRule, err := t.GetSchemaNameRule()
if err != nil {
Expand Down Expand Up @@ -275,6 +287,11 @@ type TableStruct struct {
TableIncompatibleDDL []string `json:"tableIncompatibleDDL"`
}

func (s *TableStruct) String() string {
js, _ := stringutil.MarshalIndentJSON(s)
return js
}

func IStructMigrateTableStructure(p IStructMigrateAttributesProcessor) (*TableStruct, error) {
var incompatibleSqls []string
schemaNameT, err := p.GenSchemaNameT()
Expand Down
8 changes: 4 additions & 4 deletions example/struct_migrate_task_test.toml
View file
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
task-name = "smig-task00"
datasource-name-s = "pg117"
datasource-name-t = "tidb161"
datasource-name-s = "ora151"
datasource-name-t = "tidb85"
comment = "测试数据源"

[struct-migrate-param]
Expand All @@ -24,10 +24,10 @@ case-field-rule-s = "0"
case-field-rule-t = "1"

[schema-route-rule]
schema-name-s = "marvin"
schema-name-s = "MARVIN"
schema-name-t = "ucccis"
# include-table-s 迁移所有表需指定 *，特定表写具体表名
include-table-s = ["users","marvin10"]
include-table-s = ["T10"]
exclude-table-s = []
# sequence 设置空代表不迁移 sequence
include-sequence-s = []
Expand Down

0 comments on commit 3a0c490

Please sign in to comment.