Skip to content

Commit

Permalink
Sets the charset and collation for the SQL schema to utf8mb4 (tgstati…
Browse files Browse the repository at this point in the history 
…on#56060)

Changes the charset and collation in the SQL schema to utf8mb4 (which is default in ubuntu 20.04 and debian 10 for MariaDB). This solves all the problems with russian or other non-latin1 characters in many cases related to the use of the database, because surprisingly the rest is already ready for such use cases.

I am not adding any SQL script to migrate from latin1 to utf8mb4, as this is an extremely destructive action. It must be performed manually with the data of all tables backed up before doing so.
However, this can be done by using the following queries, one time for database:
ALTER DATABASE databse_name CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
and one for each table:
ALTER TABLE table_name CONVERT TO CHARACTER SET utf8mb4 COLLATE utf8mb4_general_ci;
  • Loading branch information
Vladimir Veisman authored Jan 10, 2021
1 parent a00c2ec commit 56a01e8
Show file tree
Hide file tree
Showing 2 changed files with 38 additions and 38 deletions.
38 changes: 19 additions & 19 deletions SQL/tgstation_schema.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ CREATE TABLE `admin` (
`rank` varchar(32) NOT NULL,
`feedback` varchar(255) DEFAULT NULL,
PRIMARY KEY (`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -41,7 +41,7 @@ CREATE TABLE `admin_log` (
`target` varchar(32) NOT NULL,
`log` varchar(1000) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -57,7 +57,7 @@ CREATE TABLE `admin_ranks` (
`exclude_flags` smallint(5) unsigned NOT NULL,
`can_edit_flags` smallint(5) unsigned NOT NULL,
PRIMARY KEY (`rank`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -95,7 +95,7 @@ CREATE TABLE `ban` (
KEY `idx_ban_isbanned` (`ckey`,`role`,`unbanned_datetime`,`expiration_time`),
KEY `idx_ban_isbanned_details` (`ckey`,`ip`,`computerid`,`role`,`unbanned_datetime`,`expiration_time`),
KEY `idx_ban_count` (`bantime`,`a_ckey`,`applies_to_admins`,`unbanned_datetime`,`expiration_time`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -115,7 +115,7 @@ CREATE TABLE `connection_log` (
`ip` int(10) unsigned NOT NULL,
`computerid` varchar(45) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -152,7 +152,7 @@ CREATE TABLE `death` (
`last_words` varchar(255) DEFAULT NULL,
`suicide` tinyint(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -171,7 +171,7 @@ CREATE TABLE `feedback` (
`version` tinyint(3) unsigned NOT NULL,
`json` json NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -187,7 +187,7 @@ CREATE TABLE `ipintel` (
`intel` double NOT NULL DEFAULT '0',
PRIMARY KEY (`ip`),
KEY `idx_ipintel` (`ip`,`intel`,`date`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -206,7 +206,7 @@ CREATE TABLE `legacy_population` (
`server_port` smallint(5) unsigned NOT NULL,
`round_id` int(11) unsigned NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -231,7 +231,7 @@ CREATE TABLE `library` (
KEY `idx_lib_id_del` (`id`,`deleted`),
KEY `idx_lib_del_title` (`deleted`,`title`),
KEY `idx_lib_search` (`deleted`,`author`,`title`,`category`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -264,7 +264,7 @@ CREATE TABLE `messages` (
KEY `idx_msg_ckey_time` (`targetckey`,`timestamp`, `deleted`),
KEY `idx_msg_type_ckeys_time` (`type`,`targetckey`,`adminckey`,`timestamp`, `deleted`),
KEY `idx_msg_type_ckey_time_odr` (`type`,`targetckey`,`timestamp`, `deleted`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -301,7 +301,7 @@ CREATE TABLE IF NOT EXISTS `role_time_log` (
KEY `ckey` (`ckey`),
KEY `job` (`job`),
KEY `datetime` (`datetime`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -326,7 +326,7 @@ CREATE TABLE `player` (
PRIMARY KEY (`ckey`),
KEY `idx_player_cid_ckey` (`computerid`,`ckey`),
KEY `idx_player_ip_ckey` (`ip`,`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -349,7 +349,7 @@ CREATE TABLE `poll_option` (
`deleted` tinyint(1) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `idx_pop_pollid` (`pollid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -378,7 +378,7 @@ CREATE TABLE `poll_question` (
KEY `idx_pquest_question_time_ckey` (`question`,`starttime`,`endtime`,`createdby_ckey`,`createdby_ip`),
KEY `idx_pquest_time_deleted_id` (`starttime`,`endtime`, `deleted`, `id`),
KEY `idx_pquest_id_time_type_admin` (`id`,`starttime`,`endtime`,`polltype`,`adminonly`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -399,7 +399,7 @@ CREATE TABLE `poll_textreply` (
`deleted` tinyint(1) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `idx_ptext_pollid_ckey` (`pollid`,`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -422,7 +422,7 @@ CREATE TABLE `poll_vote` (
PRIMARY KEY (`id`),
KEY `idx_pvote_pollid_ckey` (`pollid`,`ckey`),
KEY `idx_pvote_optionid_ckey` (`optionid`,`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -447,7 +447,7 @@ CREATE TABLE `round` (
`map_name` VARCHAR(32) NULL,
`station_name` VARCHAR(80) NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

Expand All @@ -460,7 +460,7 @@ CREATE TABLE `schema_revision` (
`minor` TINYINT(3) unsigned NOT NULL,
`date` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`major`, `minor`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

--
-- Table structure for table `stickyban`
Expand Down
38 changes: 19 additions & 19 deletions SQL/tgstation_schema_prefixed.sql
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,7 +21,7 @@ CREATE TABLE `SS13_admin` (
`rank` varchar(32) NOT NULL,
`feedback` varchar(255) DEFAULT NULL,
PRIMARY KEY (`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -41,7 +41,7 @@ CREATE TABLE `SS13_admin_log` (
`target` varchar(32) NOT NULL,
`log` varchar(1000) NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -57,7 +57,7 @@ CREATE TABLE `SS13_admin_ranks` (
`exclude_flags` smallint(5) unsigned NOT NULL,
`can_edit_flags` smallint(5) unsigned NOT NULL,
PRIMARY KEY (`rank`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -95,7 +95,7 @@ CREATE TABLE `SS13_ban` (
KEY `idx_ban_isbanned` (`ckey`,`role`,`unbanned_datetime`,`expiration_time`),
KEY `idx_ban_isbanned_details` (`ckey`,`ip`,`computerid`,`role`,`unbanned_datetime`,`expiration_time`),
KEY `idx_ban_count` (`bantime`,`a_ckey`,`applies_to_admins`,`unbanned_datetime`,`expiration_time`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -115,7 +115,7 @@ CREATE TABLE `SS13_connection_log` (
`ip` int(10) unsigned NOT NULL,
`computerid` varchar(45) DEFAULT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -152,7 +152,7 @@ CREATE TABLE `SS13_death` (
`last_words` varchar(255) DEFAULT NULL,
`suicide` tinyint(1) NOT NULL DEFAULT '0',
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -171,7 +171,7 @@ CREATE TABLE `SS13_feedback` (
`key_type` enum('text', 'amount', 'tally', 'nested tally', 'associative') NOT NULL,
`json` json NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=MyISAM DEFAULT CHARSET=latin1;
) ENGINE=MyISAM DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -187,7 +187,7 @@ CREATE TABLE `SS13_ipintel` (
`intel` double NOT NULL DEFAULT '0',
PRIMARY KEY (`ip`),
KEY `idx_ipintel` (`ip`,`intel`,`date`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -206,7 +206,7 @@ CREATE TABLE `SS13_legacy_population` (
`server_port` smallint(5) unsigned NOT NULL,
`round_id` int(11) unsigned NOT NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -231,7 +231,7 @@ CREATE TABLE `SS13_library` (
KEY `idx_lib_id_del` (`id`,`deleted`),
KEY `idx_lib_del_title` (`deleted`,`title`),
KEY `idx_lib_search` (`deleted`,`author`,`title`,`category`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -264,7 +264,7 @@ CREATE TABLE `SS13_messages` (
KEY `idx_msg_ckey_time` (`targetckey`,`timestamp`, `deleted`),
KEY `idx_msg_type_ckeys_time` (`type`,`targetckey`,`adminckey`,`timestamp`, `deleted`),
KEY `idx_msg_type_ckey_time_odr` (`type`,`targetckey`,`timestamp`, `deleted`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -301,7 +301,7 @@ CREATE TABLE IF NOT EXISTS `SS13_role_time_log` (
KEY `ckey` (`ckey`),
KEY `job` (`job`),
KEY `datetime` (`datetime`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -326,7 +326,7 @@ CREATE TABLE `SS13_player` (
PRIMARY KEY (`ckey`),
KEY `idx_player_cid_ckey` (`computerid`,`ckey`),
KEY `idx_player_ip_ckey` (`ip`,`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -349,7 +349,7 @@ CREATE TABLE `SS13_poll_option` (
`deleted` tinyint(1) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `idx_pop_pollid` (`pollid`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand Down Expand Up @@ -378,7 +378,7 @@ CREATE TABLE `SS13_poll_question` (
KEY `idx_pquest_question_time_ckey` (`question`,`starttime`,`endtime`,`createdby_ckey`,`createdby_ip`),
KEY `idx_pquest_time_deleted_id` (`starttime`,`endtime`, `deleted`, `id`),
KEY `idx_pquest_id_time_type_admin` (`id`,`starttime`,`endtime`,`polltype`,`adminonly`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -399,7 +399,7 @@ CREATE TABLE `SS13_poll_textreply` (
`deleted` tinyint(1) unsigned NOT NULL DEFAULT '0',
PRIMARY KEY (`id`),
KEY `idx_ptext_pollid_ckey` (`pollid`,`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -422,7 +422,7 @@ CREATE TABLE `SS13_poll_vote` (
PRIMARY KEY (`id`),
KEY `idx_pvote_pollid_ckey` (`pollid`,`ckey`),
KEY `idx_pvote_optionid_ckey` (`optionid`,`ckey`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;

--
Expand All @@ -447,7 +447,7 @@ CREATE TABLE `SS13_round` (
`map_name` VARCHAR(32) NULL,
`station_name` VARCHAR(80) NULL,
PRIMARY KEY (`id`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;
/*!40101 SET character_set_client = @saved_cs_client */;
/*!40103 SET TIME_ZONE=@OLD_TIME_ZONE */;

Expand All @@ -460,7 +460,7 @@ CREATE TABLE `SS13_schema_revision` (
`minor` TINYINT(3) unsigned NOT NULL,
`date` DATETIME NOT NULL DEFAULT CURRENT_TIMESTAMP ON UPDATE CURRENT_TIMESTAMP,
PRIMARY KEY (`major`,`minor`)
) ENGINE=InnoDB DEFAULT CHARSET=latin1;
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_general_ci;

--
-- Table structure for table `SS13_stickyban`
Expand Down

0 comments on commit 56a01e8

Please sign in to comment.