diff --git a/.github/workflows/codeql.yml b/.github/workflows/codeql.yml new file mode 100644 index 000000000..83a3d6ee8 --- /dev/null +++ b/.github/workflows/codeql.yml @@ -0,0 +1,41 @@ +name: "CodeQL" + +on: + push: + branches: [ "master" ] + pull_request: + branches: [ "master" ] + schedule: + - cron: "18 19 * * 1" + +jobs: + analyze: + name: Analyze + runs-on: ubuntu-latest + permissions: + actions: read + contents: read + security-events: write + + strategy: + fail-fast: false + matrix: + language: [ go ] + + steps: + - name: Checkout + uses: actions/checkout@v4 + + - name: Initialize CodeQL + uses: github/codeql-action/init@v3 + with: + languages: ${{ matrix.language }} + queries: +security-and-quality + + - name: Autobuild + uses: github/codeql-action/autobuild@v3 + + - name: Perform CodeQL Analysis + uses: github/codeql-action/analyze@v3 + with: + category: "/language:${{ matrix.language }}" diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 6bf3524d4..2e07fea91 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -11,6 +11,12 @@ env: MYSQL_TEST_CONCURRENT: 1 jobs: + lint: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: dominikh/staticcheck-action@v1.3.1 + list: runs-on: ubuntu-latest outputs: @@ -20,29 +26,31 @@ jobs: id: set-matrix run: | import json + import os go = [ # Keep the most recent production release at the top - '1.17', + '1.23', # Older production releases - '1.16', - '1.15', - '1.14', - '1.13', + '1.22', + '1.21', ] mysql = [ + '9.0', + '8.4', # LTS '8.0', '5.7', - '5.6', - 'mariadb-10.6', - 'mariadb-10.5', - 'mariadb-10.4', - 'mariadb-10.3', + 'mariadb-11.4', # LTS + 'mariadb-11.2', + 'mariadb-11.1', + 'mariadb-10.11', # LTS + 'mariadb-10.6', # LTS + 'mariadb-10.5', # LTS ] includes = [] # Go versions compatibility check for v in go[1:]: - includes.append({'os': 'ubuntu-latest', 'go': v, 'mysql': mysql[0]}) + includes.append({'os': 'ubuntu-latest', 'go': v, 'mysql': mysql[0]}) matrix = { # OS vs MySQL versions @@ -53,7 +61,8 @@ jobs: 'include': includes } output = json.dumps(matrix, separators=(',', ':')) - print('::set-output name=matrix::{0}'.format(output)) + with open(os.environ["GITHUB_OUTPUT"], 'a', encoding="utf-8") as f: + print(f"matrix={output}", file=f) shell: python test: needs: list @@ -62,8 +71,8 @@ jobs: fail-fast: false matrix: ${{ fromJSON(needs.list.outputs.matrix) }} steps: - - uses: actions/checkout@v2 - - uses: actions/setup-go@v2 + - uses: actions/checkout@v4 + - uses: actions/setup-go@v5 with: go-version: ${{ matrix.go }} - uses: shogo82148/actions-setup-mysql@v1 @@ -74,17 +83,18 @@ jobs: my-cnf: | innodb_log_file_size=256MB innodb_buffer_pool_size=512MB - max_allowed_packet=16MB + max_allowed_packet=48MB ; TestConcurrent fails if max_connections is too large max_connections=50 local_infile=1 + performance_schema=on - name: setup database run: | mysql --user 'root' --host '127.0.0.1' -e 'create database gotest;' - name: test run: | - go test -v '-covermode=count' '-coverprofile=coverage.out' + go test -v '-race' '-covermode=atomic' '-coverprofile=coverage.out' -parallel 10 - name: Send coverage uses: shogo82148/actions-goveralls@v1 diff --git a/AUTHORS b/AUTHORS index 876b2964a..510b869b7 100644 --- a/AUTHORS +++ b/AUTHORS @@ -13,29 +13,39 @@ Aaron Hopkins Achille Roussel +Aidan Alex Snast Alexey Palazhchenko Andrew Reid Animesh Ray Arne Hormann Ariel Mashraki +Artur Melanchyk Asta Xie +B Lamarche +Bes Dollma +Bogdan Constantinescu +Brian Hendriks Bulat Gaifullin Caine Jette Carlos Nieto Chris Kirkland Chris Moos Craig Wilson +Daemonxiao <735462752 at qq.com> Daniel Montoya Daniel Nichter Daniël van Eeden Dave Protasowski +Dirkjan Bussink DisposaBoy Egor Smolyakov Erwan Martin +Evan Elias Evan Shaw Frederick Mayle Gustavo Kristic +Gusted Hajime Nakagami Hanno Braun Henri Yandell @@ -45,13 +55,18 @@ ICHINOSE Shogo Ilia Cimpoes INADA Naoki Jacek Szwec +Jakub Adamus James Harr Janek Vedock +Jason Ng +Jean-Yves Pellé Jeff Hodges Jeffrey Charles +Jennifer Purevsuren Jerome Meyer Jiajia Zhong Jian Zhen +Joe Mann Joshua Prunier Julien Lefevre Julien Schmidt @@ -61,6 +76,7 @@ Kamil Dziedzic Kei Kamikawa Kevin Malachowski Kieron Woodhouse +Lance Tian Lennart Rudolph Leonardo YongUk Kim Linh Tran Tuan @@ -71,17 +87,23 @@ Lunny Xiao Luke Scott Maciej Zimnoch Michael Woolnough +Nao Yokotsuka Nathanial Murphy Nicola Peduzzi +Oliver Bone Olivier Mengué oscarzhao Paul Bonser +Paulius Lozys Peter Schultz +Phil Porada +Minh Quang Rebecca Chin Reed Allman Richard Wilkes Robert Russell Runrioter Wung +Samantha Frank Santhosh Kumar Tekuri Sho Iizuka Sho Ikeda @@ -92,6 +114,7 @@ Stan Putrya Stanley Gunawan Steven Hartland Tan Jinhua <312841925 at qq.com> +Tetsuro Aoki Thomas Wodarek Tim Ruffles Tom Jenkinson @@ -101,6 +124,7 @@ Xiangyu Hu Xiaobing Jiang Xiuming Chen Xuehong Chan +Zhang Xiang Zhenye Xie Zhixin Wen Ziheng Lyu @@ -110,13 +134,19 @@ Ziheng Lyu Barracuda Networks, Inc. Counting Ltd. DigitalOcean Inc. +Dolthub Inc. +dyves labs AG Facebook Inc. GitHub Inc. Google Inc. InfoSum Ltd. Keybase Inc. +Microsoft Corp. Multiplay Ltd. Percona LLC +PingCAP Inc. Pivotal Inc. +Shattered Silicon Ltd. Stripe Inc. +ThousandEyes Zendesk Inc. diff --git a/CHANGELOG.md b/CHANGELOG.md index 72a738ed5..a1b23c66b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,3 +1,115 @@ +# Changelog + +## v1.9.1 (2025-03-21) + +### Major Changes + +* Add Charset() option. (#1679) + +### Bugfixes + +* go.mod: fix go version format (#1682) +* Fix FormatDSN missing ConnectionAttributes (#1619) + +## v1.9.0 (2025-02-18) + +### Major Changes + +- Implement zlib compression. (#1487) +- Supported Go version is updated to Go 1.21+. (#1639) +- Add support for VECTOR type introduced in MySQL 9.0. (#1609) +- Config object can have custom dial function. (#1527) + +### Bugfixes + +- Fix auth errors when username/password are too long. (#1625) +- Check if MySQL supports CLIENT_CONNECT_ATTRS before sending client attributes. (#1640) +- Fix auth switch request handling. (#1666) + +### Other changes + +- Add "filename:line" prefix to log in go-mysql. Custom loggers now show it. (#1589) +- Improve error handling. It reduces the "busy buffer" errors. (#1595, #1601, #1641) +- Use `strconv.Atoi` to parse max_allowed_packet. (#1661) +- `rejectReadOnly` option now handles ER_READ_ONLY_MODE (1290) error too. (#1660) + + +## Version 1.8.1 (2024-03-26) + +Bugfixes: + +- fix race condition when context is canceled in [#1562](https://github.com/go-sql-driver/mysql/pull/1562) and [#1570](https://github.com/go-sql-driver/mysql/pull/1570) + +## Version 1.8.0 (2024-03-09) + +Major Changes: + +- Use `SET NAMES charset COLLATE collation`. by @methane in [#1437](https://github.com/go-sql-driver/mysql/pull/1437) + - Older go-mysql-driver used `collation_id` in the handshake packet. But it caused collation mismatch in some situation. + - If you don't specify charset nor collation, go-mysql-driver sends `SET NAMES utf8mb4` for new connection. This uses server's default collation for utf8mb4. + - If you specify charset, go-mysql-driver sends `SET NAMES `. This uses the server's default collation for ``. + - If you specify collation and/or charset, go-mysql-driver sends `SET NAMES charset COLLATE collation`. +- PathEscape dbname in DSN. by @methane in [#1432](https://github.com/go-sql-driver/mysql/pull/1432) + - This is backward incompatible in rare case. Check your DSN. +- Drop Go 1.13-17 support by @methane in [#1420](https://github.com/go-sql-driver/mysql/pull/1420) + - Use Go 1.18+ +- Parse numbers on text protocol too by @methane in [#1452](https://github.com/go-sql-driver/mysql/pull/1452) + - When text protocol is used, go-mysql-driver passed bare `[]byte` to database/sql for avoid unnecessary allocation and conversion. + - If user specified `*any` to `Scan()`, database/sql passed the `[]byte` into the target variable. + - This confused users because most user doesn't know when text/binary protocol used. + - go-mysql-driver 1.8 converts integer/float values into int64/double even in text protocol. This doesn't increase allocation compared to `[]byte` and conversion cost is negatable. +- New options start using the Functional Option Pattern to avoid increasing technical debt in the Config object. Future version may introduce Functional Option for existing options, but not for now. + - Make TimeTruncate functional option by @methane in [1552](https://github.com/go-sql-driver/mysql/pull/1552) + - Add BeforeConnect callback to configuration object by @ItalyPaleAle in [#1469](https://github.com/go-sql-driver/mysql/pull/1469) + + +Other changes: + +- Adding DeregisterDialContext to prevent memory leaks with dialers we don't need anymore by @jypelle in https://github.com/go-sql-driver/mysql/pull/1422 +- Make logger configurable per connection by @frozenbonito in https://github.com/go-sql-driver/mysql/pull/1408 +- Fix ColumnType.DatabaseTypeName for mediumint unsigned by @evanelias in https://github.com/go-sql-driver/mysql/pull/1428 +- Add connection attributes by @Daemonxiao in https://github.com/go-sql-driver/mysql/pull/1389 +- Stop `ColumnTypeScanType()` from returning `sql.RawBytes` by @methane in https://github.com/go-sql-driver/mysql/pull/1424 +- Exec() now provides access to status of multiple statements. by @mherr-google in https://github.com/go-sql-driver/mysql/pull/1309 +- Allow to change (or disable) the default driver name for registration by @dolmen in https://github.com/go-sql-driver/mysql/pull/1499 +- Add default connection attribute '_server_host' by @oblitorum in https://github.com/go-sql-driver/mysql/pull/1506 +- QueryUnescape DSN ConnectionAttribute value by @zhangyangyu in https://github.com/go-sql-driver/mysql/pull/1470 +- Add client_ed25519 authentication by @Gusted in https://github.com/go-sql-driver/mysql/pull/1518 + +## Version 1.7.1 (2023-04-25) + +Changes: + + - bump actions/checkout@v3 and actions/setup-go@v3 (#1375) + - Add go1.20 and mariadb10.11 to the testing matrix (#1403) + - Increase default maxAllowedPacket size. (#1411) + +Bugfixes: + + - Use SET syntax as specified in the MySQL documentation (#1402) + + +## Version 1.7 (2022-11-29) + +Changes: + + - Drop support of Go 1.12 (#1211) + - Refactoring `(*textRows).readRow` in a more clear way (#1230) + - util: Reduce boundary check in escape functions. (#1316) + - enhancement for mysqlConn handleAuthResult (#1250) + +New Features: + + - support Is comparison on MySQLError (#1210) + - return unsigned in database type name when necessary (#1238) + - Add API to express like a --ssl-mode=PREFERRED MySQL client (#1370) + - Add SQLState to MySQLError (#1321) + +Bugfixes: + + - Fix parsing 0 year. (#1257) + + ## Version 1.6 (2021-04-01) Changes: @@ -128,7 +240,7 @@ New Features: - Enable microsecond resolution on TIME, DATETIME and TIMESTAMP (#249) - Support for returning table alias on Columns() (#289, #359, #382) - - Placeholder interpolation, can be actived with the DSN parameter `interpolateParams=true` (#309, #318, #490) + - Placeholder interpolation, can be activated with the DSN parameter `interpolateParams=true` (#309, #318, #490) - Support for uint64 parameters with high bit set (#332, #345) - Cleartext authentication plugin support (#327) - Exported ParseDSN function and the Config struct (#403, #419, #429) @@ -172,7 +284,7 @@ Changes: - Also exported the MySQLWarning type - mysqlConn.Close returns the first error encountered instead of ignoring all errors - writePacket() automatically writes the packet size to the header - - readPacket() uses an iterative approach instead of the recursive approach to merge splitted packets + - readPacket() uses an iterative approach instead of the recursive approach to merge split packets New Features: @@ -220,7 +332,7 @@ Bugfixes: - Fixed MySQL 4.1 support: MySQL 4.1 sends packets with lengths which differ from the specification - Convert to DB timezone when inserting `time.Time` - - Splitted packets (more than 16MB) are now merged correctly + - Split packets (more than 16MB) are now merged correctly - Fixed false positive `io.EOF` errors when the data was fully read - Avoid panics on reuse of closed connections - Fixed empty string producing false nil values diff --git a/README.md b/README.md index ded6e3b16..da4593ccf 100644 --- a/README.md +++ b/README.md @@ -38,17 +38,26 @@ A MySQL-Driver for Go's [database/sql](https://golang.org/pkg/database/sql/) pac * Secure `LOAD DATA LOCAL INFILE` support with file allowlisting and `io.Reader` support * Optional `time.Time` parsing * Optional placeholder interpolation + * Supports zlib compression. ## Requirements - * Go 1.13 or higher. We aim to support the 3 latest versions of Go. - * MySQL (4.1+), MariaDB, Percona Server, Google CloudSQL or Sphinx (2.2.3+) + +* Go 1.21 or higher. We aim to support the 3 latest versions of Go. +* MySQL (5.7+) and MariaDB (10.5+) are supported. +* [TiDB](https://github.com/pingcap/tidb) is supported by PingCAP. + * Do not ask questions about TiDB in our issue tracker or forum. + * [Document](https://docs.pingcap.com/tidb/v6.1/dev-guide-sample-application-golang) + * [Forum](https://ask.pingcap.com/) +* go-mysql would work with Percona Server, Google CloudSQL or Sphinx (2.2.3+). + * Maintainers won't support them. Do not expect issues are investigated and resolved by maintainers. + * Investigate issues yourself and please send a pull request to fix it. --------------------------------------- ## Installation Simple install the package to your [$GOPATH](https://github.com/golang/go/wiki/GOPATH "GOPATH") with the [go tool](https://golang.org/cmd/go/ "go command") from shell: ```bash -$ go get -u github.com/go-sql-driver/mysql +go get -u github.com/go-sql-driver/mysql ``` Make sure [Git is installed](https://git-scm.com/downloads) on your machine and in your system's `PATH`. @@ -114,6 +123,12 @@ This has the same effect as an empty DSN string: ``` +`dbname` is escaped by [PathEscape()](https://pkg.go.dev/net/url#PathEscape) since v1.8.0. If your database name is `dbname/withslash`, it becomes: + +``` +/dbname%2Fwithslash +``` + Alternatively, [Config.FormatDSN](https://godoc.org/github.com/go-sql-driver/mysql#Config.FormatDSN) can be used to create a DSN string by filling a struct. #### Password @@ -121,7 +136,7 @@ Passwords can consist of any character. Escaping is **not** necessary. #### Protocol See [net.Dial](https://golang.org/pkg/net/#Dial) for more information which networks are available. -In general you should use an Unix domain socket if available and TCP otherwise for best performance. +In general you should use a Unix domain socket if available and TCP otherwise for best performance. #### Address For TCP and UDP networks, addresses have the form `host[:port]`. @@ -145,7 +160,7 @@ Default: false ``` `allowAllFiles=true` disables the file allowlist for `LOAD DATA LOCAL INFILE` and allows *all* files. -[*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html) +[*Might be insecure!*](https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-local) ##### `allowCleartextPasswords` @@ -157,6 +172,17 @@ Default: false `allowCleartextPasswords=true` allows using the [cleartext client side plugin](https://dev.mysql.com/doc/en/cleartext-pluggable-authentication.html) if required by an account, such as one defined with the [PAM authentication plugin](http://dev.mysql.com/doc/en/pam-authentication-plugin.html). Sending passwords in clear text may be a security problem in some configurations. To avoid problems if there is any possibility that the password would be intercepted, clients should connect to MySQL Server using a method that protects the password. Possibilities include [TLS / SSL](#tls), IPsec, or a private network. + +##### `allowFallbackToPlaintext` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +`allowFallbackToPlaintext=true` acts like a `--ssl-mode=PREFERRED` MySQL client as described in [Command Options for Connecting to the Server](https://dev.mysql.com/doc/refman/5.7/en/connection-options.html#option_general_ssl-mode) + ##### `allowNativePasswords` ``` @@ -183,10 +209,9 @@ Valid Values: Default: none ``` -Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset failes. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). +Sets the charset used for client-server interaction (`"SET NAMES "`). If multiple charsets are set (separated by a comma), the following charset is used if setting the charset fails. This enables for example support for `utf8mb4` ([introduced in MySQL 5.5.3](http://dev.mysql.com/doc/refman/5.5/en/charset-unicode-utf8mb4.html)) with fallback to `utf8` for older servers (`charset=utf8mb4,utf8`). -Usage of the `charset` parameter is discouraged because it issues additional queries to the server. -Unless you need the fallback behavior, please use `collation` instead. +See also [Unicode Support](#unicode-support). ##### `checkConnLiveness` @@ -215,6 +240,7 @@ The default collation (`utf8mb4_general_ci`) is supported from MySQL 5.5. You s Collations for charset "ucs2", "utf16", "utf16le", and "utf32" can not be used ([ref](https://dev.mysql.com/doc/refman/5.7/en/charset-connection.html#charset-connection-impermissible-client-charset)). +See also [Unicode Support](#unicode-support). ##### `clientFoundRows` @@ -242,6 +268,16 @@ SELECT u.id FROM users as u will return `u.id` instead of just `id` if `columnsWithAlias=true`. +##### `compress` + +``` +Type: bool +Valid Values: true, false +Default: false +``` + +Toggles zlib compression. false by default. + ##### `interpolateParams` ``` @@ -268,13 +304,22 @@ Note that this sets the location for time.Time values but does not change MySQL' Please keep in mind, that param values must be [url.QueryEscape](https://golang.org/pkg/net/url/#QueryEscape)'ed. Alternatively you can manually replace the `/` with `%2F`. For example `US/Pacific` would be `loc=US%2FPacific`. +##### `timeTruncate` + +``` +Type: duration +Default: 0 +``` + +[Truncate time values](https://pkg.go.dev/time#Duration.Truncate) to the specified duration. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. + ##### `maxAllowedPacket` ``` Type: decimal number -Default: 4194304 +Default: 64*1024*1024 ``` -Max packet size allowed in bytes. The default value is 4 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*. +Max packet size allowed in bytes. The default value is 64 MiB and should be adjusted to match the server settings. `maxAllowedPacket=0` can be used to automatically fetch the `max_allowed_packet` variable from server *on every connection*. ##### `multiStatements` @@ -284,9 +329,25 @@ Valid Values: true, false Default: false ``` -Allow multiple statements in one query. While this allows batch queries, it also greatly increases the risk of SQL injections. Only the result of the first query is returned, all other results are silently discarded. +Allow multiple statements in one query. This can be used to bach multiple queries. Use [Rows.NextResultSet()](https://pkg.go.dev/database/sql#Rows.NextResultSet) to get result of the second and subsequent queries. + +When `multiStatements` is used, `?` parameters must only be used in the first statement. [interpolateParams](#interpolateparams) can be used to avoid this limitation unless prepared statement is used explicitly. -When `multiStatements` is used, `?` parameters must only be used in the first statement. +It's possible to access the last inserted ID and number of affected rows for multiple statements by using `sql.Conn.Raw()` and the `mysql.Result`. For example: + +```go +conn, _ := db.Conn(ctx) +conn.Raw(func(conn any) error { + ex := conn.(driver.Execer) + res, err := ex.Exec(` + UPDATE point SET x = 1 WHERE y = 2; + UPDATE point SET x = 2 WHERE y = 3; + `, nil) + // Both slices have 2 elements. + log.Print(res.(mysql.Result).AllRowsAffected()) + log.Print(res.(mysql.Result).AllLastInsertIds()) +}) +``` ##### `parseTime` @@ -382,6 +443,15 @@ Default: 0 I/O write timeout. The value must be a decimal number with a unit suffix (*"ms"*, *"s"*, *"m"*, *"h"*), such as *"30s"*, *"0.5m"* or *"1m30s"*. +##### `connectionAttributes` + +``` +Type: comma-delimited string of user-defined "key:value" pairs +Valid Values: (:,:,...) +Default: none +``` + +[Connection attributes](https://dev.mysql.com/doc/refman/8.0/en/performance-schema-connection-attribute-tables.html) are key-value pairs that application programs can pass to the server at connect time. ##### System Variables @@ -454,12 +524,15 @@ user:password@/ The connection pool is managed by Go's database/sql package. For details on how to configure the size of the pool and how long connections stay in the pool see `*DB.SetMaxOpenConns`, `*DB.SetMaxIdleConns`, and `*DB.SetConnMaxLifetime` in the [database/sql documentation](https://golang.org/pkg/database/sql/). The read, write, and dial timeouts for each individual connection are configured with the DSN parameters [`readTimeout`](#readtimeout), [`writeTimeout`](#writetimeout), and [`timeout`](#timeout), respectively. ## `ColumnType` Support -This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. All Unsigned database type names will be returned `UNSIGNED ` with `INT`, `TINYINT`, `SMALLINT`, `BIGINT`. +This driver supports the [`ColumnType` interface](https://golang.org/pkg/database/sql/#ColumnType) introduced in Go 1.8, with the exception of [`ColumnType.Length()`](https://golang.org/pkg/database/sql/#ColumnType.Length), which is currently not supported. All Unsigned database type names will be returned `UNSIGNED ` with `INT`, `TINYINT`, `SMALLINT`, `MEDIUMINT`, `BIGINT`. ## `context.Context` Support Go 1.8 added `database/sql` support for `context.Context`. This driver supports query timeouts and cancellation via contexts. See [context support in the database/sql package](https://golang.org/doc/go1.8#database_sql) for more details. +> [!IMPORTANT] +> The `QueryContext`, `ExecContext`, etc. variants provided by `database/sql` will cause the connection to be closed if the provided context is cancelled or timed out before the result is received by the driver. + ### `LOAD DATA LOCAL INFILE` support For this feature you need direct access to the package. Therefore you must change the import path (no `_`): @@ -467,7 +540,7 @@ For this feature you need direct access to the package. Therefore you must chang import "github.com/go-sql-driver/mysql" ``` -Files must be explicitly allowed by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the allowlist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](http://dev.mysql.com/doc/refman/5.7/en/load-data-local.html)). +Files must be explicitly allowed by registering them with `mysql.RegisterLocalFile(filepath)` (recommended) or the allowlist check must be deactivated by using the DSN parameter `allowAllFiles=true` ([*Might be insecure!*](https://dev.mysql.com/doc/refman/8.0/en/load-data.html#load-data-local)). To use a `io.Reader` a handler function must be registered with `mysql.RegisterReaderHandler(name, handler)` which returns a `io.Reader` or `io.ReadCloser`. The Reader is available with the filepath `Reader::` then. Choose different names for different handlers and `DeregisterReaderHandler` when you don't need it anymore. @@ -485,9 +558,11 @@ However, many want to scan MySQL `DATE` and `DATETIME` values into `time.Time` v ### Unicode support Since version 1.5 Go-MySQL-Driver automatically uses the collation ` utf8mb4_general_ci` by default. -Other collations / charsets can be set using the [`collation`](#collation) DSN parameter. +Other charsets / collations can be set using the [`charset`](#charset) or [`collation`](#collation) DSN parameter. -Version 1.0 of the driver recommended adding `&charset=utf8` (alias for `SET NAMES utf8`) to the DSN to enable proper UTF-8 support. This is not necessary anymore. The [`collation`](#collation) parameter should be preferred to set another collation / charset than the default. +- When only the `charset` is specified, the `SET NAMES ` query is sent and the server's default collation is used. +- When both the `charset` and `collation` are specified, the `SET NAMES COLLATE ` query is sent. +- When only the `collation` is specified, the collation is specified in the protocol handshake and the `SET NAMES` query is not sent. This can save one roundtrip, but note that the server may ignore the specified collation silently and use the server's default charset/collation instead. See http://dev.mysql.com/doc/refman/8.0/en/charset-unicode.html for more details on MySQL's Unicode support. diff --git a/auth.go b/auth.go index f610b5f49..74e1bd03e 100644 --- a/auth.go +++ b/auth.go @@ -13,10 +13,13 @@ import ( "crypto/rsa" "crypto/sha1" "crypto/sha256" + "crypto/sha512" "crypto/x509" "encoding/pem" "fmt" "sync" + + "filippo.io/edwards25519" ) // server pub keys registry @@ -33,27 +36,26 @@ var ( // Note: The provided rsa.PublicKey instance is exclusively owned by the driver // after registering it and may not be modified. // -// data, err := ioutil.ReadFile("mykey.pem") -// if err != nil { -// log.Fatal(err) -// } -// -// block, _ := pem.Decode(data) -// if block == nil || block.Type != "PUBLIC KEY" { -// log.Fatal("failed to decode PEM block containing public key") -// } +// data, err := os.ReadFile("mykey.pem") +// if err != nil { +// log.Fatal(err) +// } // -// pub, err := x509.ParsePKIXPublicKey(block.Bytes) -// if err != nil { -// log.Fatal(err) -// } +// block, _ := pem.Decode(data) +// if block == nil || block.Type != "PUBLIC KEY" { +// log.Fatal("failed to decode PEM block containing public key") +// } // -// if rsaPubKey, ok := pub.(*rsa.PublicKey); ok { -// mysql.RegisterServerPubKey("mykey", rsaPubKey) -// } else { -// log.Fatal("not a RSA public key") -// } +// pub, err := x509.ParsePKIXPublicKey(block.Bytes) +// if err != nil { +// log.Fatal(err) +// } // +// if rsaPubKey, ok := pub.(*rsa.PublicKey); ok { +// mysql.RegisterServerPubKey("mykey", rsaPubKey) +// } else { +// log.Fatal("not a RSA public key") +// } func RegisterServerPubKey(name string, pubKey *rsa.PublicKey) { serverPubKeyLock.Lock() if serverPubKeyRegistry == nil { @@ -226,6 +228,44 @@ func encryptPassword(password string, seed []byte, pub *rsa.PublicKey) ([]byte, return rsa.EncryptOAEP(sha1, rand.Reader, pub, plain, nil) } +// authEd25519 does ed25519 authentication used by MariaDB. +func authEd25519(scramble []byte, password string) ([]byte, error) { + // Derived from https://github.com/MariaDB/server/blob/d8e6bb00888b1f82c031938f4c8ac5d97f6874c3/plugin/auth_ed25519/ref10/sign.c + // Code style is from https://cs.opensource.google/go/go/+/refs/tags/go1.21.5:src/crypto/ed25519/ed25519.go;l=207 + h := sha512.Sum512([]byte(password)) + + s, err := edwards25519.NewScalar().SetBytesWithClamping(h[:32]) + if err != nil { + return nil, err + } + A := (&edwards25519.Point{}).ScalarBaseMult(s) + + mh := sha512.New() + mh.Write(h[32:]) + mh.Write(scramble) + messageDigest := mh.Sum(nil) + r, err := edwards25519.NewScalar().SetUniformBytes(messageDigest) + if err != nil { + return nil, err + } + + R := (&edwards25519.Point{}).ScalarBaseMult(r) + + kh := sha512.New() + kh.Write(R.Bytes()) + kh.Write(A.Bytes()) + kh.Write(scramble) + hramDigest := kh.Sum(nil) + k, err := edwards25519.NewScalar().SetUniformBytes(hramDigest) + if err != nil { + return nil, err + } + + S := k.MultiplyAdd(k, s, r) + + return append(R.Bytes(), S.Bytes()...), nil +} + func (mc *mysqlConn) sendEncryptedPassword(seed []byte, pub *rsa.PublicKey) error { enc, err := encryptPassword(mc.cfg.Passwd, seed, pub) if err != nil { @@ -276,7 +316,7 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { } // unlike caching_sha2_password, sha256_password does not accept // cleartext password on unix transport. - if mc.cfg.tls != nil { + if mc.cfg.TLS != nil { // write cleartext auth packet return append([]byte(mc.cfg.Passwd), 0), nil } @@ -291,8 +331,14 @@ func (mc *mysqlConn) auth(authData []byte, plugin string) ([]byte, error) { enc, err := encryptPassword(mc.cfg.Passwd, authData, pubKey) return enc, err + case "client_ed25519": + if len(authData) != 32 { + return nil, ErrMalformPkt + } + return authEd25519(authData, mc.cfg.Passwd) + default: - errLog.Print("unknown auth plugin:", plugin) + mc.log("unknown auth plugin:", plugin) return nil, ErrUnknownPlugin } } @@ -339,7 +385,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { switch plugin { - // https://insidemysql.com/preparing-your-community-connector-for-mysql-8-part-2-sha256/ + // https://dev.mysql.com/blog-archive/preparing-your-community-connector-for-mysql-8-part-2-sha256/ case "caching_sha2_password": switch len(authData) { case 0: @@ -347,12 +393,12 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { case 1: switch authData[0] { case cachingSha2PasswordFastAuthSuccess: - if err = mc.readResultOK(); err == nil { + if err = mc.resultUnchanged().readResultOK(); err == nil { return nil // auth successful } case cachingSha2PasswordPerformFullAuthentication: - if mc.cfg.tls != nil || mc.cfg.Net == "unix" { + if mc.cfg.TLS != nil || mc.cfg.Net == "unix" { // write cleartext auth packet err = mc.writeAuthSwitchPacket(append([]byte(mc.cfg.Passwd), 0)) if err != nil { @@ -367,16 +413,23 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { return err } data[4] = cachingSha2PasswordRequestPublicKey - mc.writePacket(data) + err = mc.writePacket(data) + if err != nil { + return err + } - // parse public key if data, err = mc.readPacket(); err != nil { return err } + if data[0] != iAuthMoreData { + return fmt.Errorf("unexpected resp from server for caching_sha2_password, perform full authentication") + } + + // parse public key block, rest := pem.Decode(data[1:]) if block == nil { - return fmt.Errorf("No Pem data found, data: %s", rest) + return fmt.Errorf("no pem data found, data: %s", rest) } pkix, err := x509.ParsePKIXPublicKey(block.Bytes) if err != nil { @@ -391,7 +444,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { return err } } - return mc.readResultOK() + return mc.resultUnchanged().readResultOK() default: return ErrMalformPkt @@ -406,6 +459,10 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { return nil // auth successful default: block, _ := pem.Decode(authData) + if block == nil { + return fmt.Errorf("no Pem data found, data: %s", authData) + } + pub, err := x509.ParsePKIXPublicKey(block.Bytes) if err != nil { return err @@ -416,7 +473,7 @@ func (mc *mysqlConn) handleAuthResult(oldAuthData []byte, plugin string) error { if err != nil { return err } - return mc.readResultOK() + return mc.resultUnchanged().readResultOK() } default: diff --git a/auth_test.go b/auth_test.go index 3bce7fe22..46e1e3b4e 100644 --- a/auth_test.go +++ b/auth_test.go @@ -291,7 +291,7 @@ func TestAuthFastCachingSHA256PasswordFullSecure(t *testing.T) { // Hack to make the caching_sha2_password plugin believe that the connection // is secure - mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + mc.cfg.TLS = &tls.Config{InsecureSkipVerify: true} // check written auth response authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 @@ -663,7 +663,7 @@ func TestAuthFastSHA256PasswordSecure(t *testing.T) { // hack to make the caching_sha2_password plugin believe that the connection // is secure - mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + mc.cfg.TLS = &tls.Config{InsecureSkipVerify: true} authData := []byte{6, 81, 96, 114, 14, 42, 50, 30, 76, 47, 1, 95, 126, 81, 62, 94, 83, 80, 52, 85} @@ -676,7 +676,7 @@ func TestAuthFastSHA256PasswordSecure(t *testing.T) { } // unset TLS config to prevent the actual establishment of a TLS wrapper - mc.cfg.tls = nil + mc.cfg.TLS = nil err = mc.writeHandshakeResponsePacket(authResp, plugin) if err != nil { @@ -734,9 +734,9 @@ func TestAuthSwitchCachingSHA256PasswordCached(t *testing.T) { expectedReply := []byte{ // 1. Packet: Hash - 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, - 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, - 153, 9, 130, + 32, 0, 0, 3, 219, 72, 64, 97, 56, 197, 167, 203, 64, 236, 168, 80, 223, + 56, 103, 217, 196, 176, 124, 60, 253, 41, 195, 10, 205, 190, 177, 206, 63, + 118, 211, 69, } if !bytes.Equal(conn.written, expectedReply) { t.Errorf("got unexpected data: %v", conn.written) @@ -803,9 +803,9 @@ func TestAuthSwitchCachingSHA256PasswordFullRSA(t *testing.T) { expectedReplyPrefix := []byte{ // 1. Packet: Hash - 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, - 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, - 153, 9, 130, + 32, 0, 0, 3, 219, 72, 64, 97, 56, 197, 167, 203, 64, 236, 168, 80, 223, + 56, 103, 217, 196, 176, 124, 60, 253, 41, 195, 10, 205, 190, 177, 206, 63, + 118, 211, 69, // 2. Packet: Pub Key Request 1, 0, 0, 5, 2, @@ -848,9 +848,9 @@ func TestAuthSwitchCachingSHA256PasswordFullRSAWithKey(t *testing.T) { expectedReplyPrefix := []byte{ // 1. Packet: Hash - 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, - 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, - 153, 9, 130, + 32, 0, 0, 3, 219, 72, 64, 97, 56, 197, 167, 203, 64, 236, 168, 80, 223, + 56, 103, 217, 196, 176, 124, 60, 253, 41, 195, 10, 205, 190, 177, 206, 63, + 118, 211, 69, // 2. Packet: Encrypted Password 0, 1, 0, 5, // [changing bytes] @@ -866,7 +866,7 @@ func TestAuthSwitchCachingSHA256PasswordFullSecure(t *testing.T) { // Hack to make the caching_sha2_password plugin believe that the connection // is secure - mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + mc.cfg.TLS = &tls.Config{InsecureSkipVerify: true} // auth switch request conn.data = []byte{44, 0, 0, 2, 254, 99, 97, 99, 104, 105, 110, 103, 95, @@ -891,9 +891,9 @@ func TestAuthSwitchCachingSHA256PasswordFullSecure(t *testing.T) { expectedReply := []byte{ // 1. Packet: Hash - 32, 0, 0, 3, 129, 93, 132, 95, 114, 48, 79, 215, 128, 62, 193, 118, 128, - 54, 75, 208, 159, 252, 227, 215, 129, 15, 242, 97, 19, 159, 31, 20, 58, - 153, 9, 130, + 32, 0, 0, 3, 219, 72, 64, 97, 56, 197, 167, 203, 64, 236, 168, 80, 223, + 56, 103, 217, 196, 176, 124, 60, 253, 41, 195, 10, 205, 190, 177, 206, 63, + 118, 211, 69, // 2. Packet: Cleartext password 7, 0, 0, 5, 115, 101, 99, 114, 101, 116, 0, @@ -1299,7 +1299,7 @@ func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { // Hack to make the caching_sha2_password plugin believe that the connection // is secure - mc.cfg.tls = &tls.Config{InsecureSkipVerify: true} + mc.cfg.TLS = &tls.Config{InsecureSkipVerify: true} // auth switch request conn.data = []byte{38, 0, 0, 2, 254, 115, 104, 97, 50, 53, 54, 95, 112, 97, @@ -1328,3 +1328,54 @@ func TestAuthSwitchSHA256PasswordSecure(t *testing.T) { t.Errorf("got unexpected data: %v", conn.written) } } + +// Derived from https://github.com/MariaDB/server/blob/6b2287fff23fbdc362499501c562f01d0d2db52e/plugin/auth_ed25519/ed25519-t.c +func TestEd25519Auth(t *testing.T) { + conn, mc := newRWMockConn(1) + mc.cfg.User = "root" + mc.cfg.Passwd = "foobar" + + authData := []byte("AAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAA") + plugin := "client_ed25519" + + // Send Client Authentication Packet + authResp, err := mc.auth(authData, plugin) + if err != nil { + t.Fatal(err) + } + err = mc.writeHandshakeResponsePacket(authResp, plugin) + if err != nil { + t.Fatal(err) + } + + // check written auth response + authRespStart := 4 + 4 + 4 + 1 + 23 + len(mc.cfg.User) + 1 + authRespEnd := authRespStart + 1 + len(authResp) + writtenAuthRespLen := conn.written[authRespStart] + writtenAuthResp := conn.written[authRespStart+1 : authRespEnd] + expectedAuthResp := []byte{ + 232, 61, 201, 63, 67, 63, 51, 53, 86, 73, 238, 35, 170, 117, 146, + 214, 26, 17, 35, 9, 8, 132, 245, 141, 48, 99, 66, 58, 36, 228, 48, + 84, 115, 254, 187, 168, 88, 162, 249, 57, 35, 85, 79, 238, 167, 106, + 68, 117, 56, 135, 171, 47, 20, 14, 133, 79, 15, 229, 124, 160, 176, + 100, 138, 14, + } + if writtenAuthRespLen != 64 { + t.Fatalf("expected 64 bytes from client, got %d", writtenAuthRespLen) + } + if !bytes.Equal(writtenAuthResp, expectedAuthResp) { + t.Fatalf("auth response did not match expected value:\n%v\n%v", writtenAuthResp, expectedAuthResp) + } + conn.written = nil + + // auth response + conn.data = []byte{ + 7, 0, 0, 2, 0, 0, 0, 2, 0, 0, 0, // OK + } + conn.maxReads = 1 + + // Handle response to auth packet + if err := mc.handleAuthResult(authData, plugin); err != nil { + t.Errorf("got error: %v", err) + } +} diff --git a/benchmark_test.go b/benchmark_test.go index 1030ddc52..5c9a046b5 100644 --- a/benchmark_test.go +++ b/benchmark_test.go @@ -46,9 +46,13 @@ func (tb *TB) checkStmt(stmt *sql.Stmt, err error) *sql.Stmt { return stmt } -func initDB(b *testing.B, queries ...string) *sql.DB { +func initDB(b *testing.B, useCompression bool, queries ...string) *sql.DB { tb := (*TB)(b) - db := tb.checkDB(sql.Open("mysql", dsn)) + comprStr := "" + if useCompression { + comprStr = "&compress=1" + } + db := tb.checkDB(sql.Open(driverNameTest, dsn+comprStr)) for _, query := range queries { if _, err := db.Exec(query); err != nil { b.Fatalf("error on %q: %v", query, err) @@ -60,10 +64,18 @@ func initDB(b *testing.B, queries ...string) *sql.DB { const concurrencyLevel = 10 func BenchmarkQuery(b *testing.B) { + benchmarkQueryHelper(b, false) +} + +func BenchmarkQueryCompression(b *testing.B) { + benchmarkQueryHelper(b, true) +} + +func benchmarkQueryHelper(b *testing.B, compr bool) { tb := (*TB)(b) b.StopTimer() b.ReportAllocs() - db := initDB(b, + db := initDB(b, compr, "DROP TABLE IF EXISTS foo", "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", `INSERT INTO foo VALUES (1, "one")`, @@ -105,7 +117,7 @@ func BenchmarkExec(b *testing.B) { tb := (*TB)(b) b.StopTimer() b.ReportAllocs() - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) db.SetMaxIdleConns(concurrencyLevel) defer db.Close() @@ -151,7 +163,7 @@ func BenchmarkRoundtripTxt(b *testing.B) { sampleString := string(sample) b.ReportAllocs() tb := (*TB)(b) - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) defer db.Close() b.StartTimer() var result string @@ -184,7 +196,7 @@ func BenchmarkRoundtripBin(b *testing.B) { sample, min, max := initRoundtripBenchmarks() b.ReportAllocs() tb := (*TB)(b) - db := tb.checkDB(sql.Open("mysql", dsn)) + db := tb.checkDB(sql.Open(driverNameTest, dsn)) defer db.Close() stmt := tb.checkStmt(db.Prepare("SELECT ?")) defer stmt.Close() @@ -222,7 +234,7 @@ func BenchmarkInterpolation(b *testing.B) { }, maxAllowedPacket: maxPacketSize, maxWriteSize: maxPacketSize - 1, - buf: newBuffer(nil), + buf: newBuffer(), } args := []driver.Value{ @@ -269,7 +281,7 @@ func benchmarkQueryContext(b *testing.B, db *sql.DB, p int) { } func BenchmarkQueryContext(b *testing.B) { - db := initDB(b, + db := initDB(b, false, "DROP TABLE IF EXISTS foo", "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", `INSERT INTO foo VALUES (1, "one")`, @@ -305,7 +317,7 @@ func benchmarkExecContext(b *testing.B, db *sql.DB, p int) { } func BenchmarkExecContext(b *testing.B) { - db := initDB(b, + db := initDB(b, false, "DROP TABLE IF EXISTS foo", "CREATE TABLE foo (id INT PRIMARY KEY, val CHAR(50))", `INSERT INTO foo VALUES (1, "one")`, @@ -314,7 +326,7 @@ func BenchmarkExecContext(b *testing.B) { defer db.Close() for _, p := range []int{1, 2, 3, 4} { b.Run(fmt.Sprintf("%d", p), func(b *testing.B) { - benchmarkQueryContext(b, db, p) + benchmarkExecContext(b, db, p) }) } } @@ -323,7 +335,7 @@ func BenchmarkExecContext(b *testing.B) { // "size=" means size of each blobs. func BenchmarkQueryRawBytes(b *testing.B) { var sizes []int = []int{100, 1000, 2000, 4000, 8000, 12000, 16000, 32000, 64000, 256000} - db := initDB(b, + db := initDB(b, false, "DROP TABLE IF EXISTS bench_rawbytes", "CREATE TABLE bench_rawbytes (id INT PRIMARY KEY, val LONGBLOB)", ) @@ -372,3 +384,59 @@ func BenchmarkQueryRawBytes(b *testing.B) { }) } } + +// BenchmarkReceiveMassiveRows measures performance of receiving large number of rows. +func BenchmarkReceiveMassiveRows(b *testing.B) { + // Setup -- prepare 10000 rows. + db := initDB(b, false, + "DROP TABLE IF EXISTS foo", + "CREATE TABLE foo (id INT PRIMARY KEY, val TEXT)") + defer db.Close() + + sval := strings.Repeat("x", 50) + stmt, err := db.Prepare(`INSERT INTO foo (id, val) VALUES (?, ?)` + strings.Repeat(",(?,?)", 99)) + if err != nil { + b.Errorf("failed to prepare query: %v", err) + return + } + for i := 0; i < 10000; i += 100 { + args := make([]any, 200) + for j := 0; j < 100; j++ { + args[j*2] = i + j + args[j*2+1] = sval + } + _, err := stmt.Exec(args...) + if err != nil { + b.Error(err) + return + } + } + stmt.Close() + + // Use b.Run() to skip expensive setup. + b.Run("query", func(b *testing.B) { + b.ReportAllocs() + + for i := 0; i < b.N; i++ { + rows, err := db.Query(`SELECT id, val FROM foo`) + if err != nil { + b.Errorf("failed to select: %v", err) + return + } + for rows.Next() { + var i int + var s sql.RawBytes + err = rows.Scan(&i, &s) + if err != nil { + b.Errorf("failed to scan: %v", err) + _ = rows.Close() + return + } + } + if err = rows.Err(); err != nil { + b.Errorf("failed to read rows: %v", err) + } + _ = rows.Close() + } + }) +} diff --git a/buffer.go b/buffer.go index 0774c5c8c..a65324315 100644 --- a/buffer.go +++ b/buffer.go @@ -10,54 +10,42 @@ package mysql import ( "io" - "net" - "time" ) const defaultBufSize = 4096 const maxCachedBufSize = 256 * 1024 +// readerFunc is a function that compatible with io.Reader. +// We use this function type instead of io.Reader because we want to +// just pass mc.readWithTimeout. +type readerFunc func([]byte) (int, error) + // A buffer which is used for both reading and writing. // This is possible since communication on each connection is synchronous. // In other words, we can't write and read simultaneously on the same connection. // The buffer is similar to bufio.Reader / Writer but zero-copy-ish // Also highly optimized for this particular use case. -// This buffer is backed by two byte slices in a double-buffering scheme type buffer struct { - buf []byte // buf is a byte buffer who's length and capacity are equal. - nc net.Conn - idx int - length int - timeout time.Duration - dbuf [2][]byte // dbuf is an array with the two byte slices that back this buffer - flipcnt uint // flipccnt is the current buffer counter for double-buffering + buf []byte // read buffer. + cachedBuf []byte // buffer that will be reused. len(cachedBuf) <= maxCachedBufSize. } // newBuffer allocates and returns a new buffer. -func newBuffer(nc net.Conn) buffer { - fg := make([]byte, defaultBufSize) +func newBuffer() buffer { return buffer{ - buf: fg, - nc: nc, - dbuf: [2][]byte{fg, nil}, + cachedBuf: make([]byte, defaultBufSize), } } -// flip replaces the active buffer with the background buffer -// this is a delayed flip that simply increases the buffer counter; -// the actual flip will be performed the next time we call `buffer.fill` -func (b *buffer) flip() { - b.flipcnt += 1 +// busy returns true if the read buffer is not empty. +func (b *buffer) busy() bool { + return len(b.buf) > 0 } -// fill reads into the buffer until at least _need_ bytes are in it -func (b *buffer) fill(need int) error { - n := b.length - // fill data into its double-buffering target: if we've called - // flip on this buffer, we'll be copying to the background buffer, - // and then filling it with network data; otherwise we'll just move - // the contents of the current buffer to the front before filling it - dest := b.dbuf[b.flipcnt&1] +// fill reads into the read buffer until at least _need_ bytes are in it. +func (b *buffer) fill(need int, r readerFunc) error { + // we'll move the contents of the current buffer to dest before filling it. + dest := b.cachedBuf // grow buffer if necessary to fit the whole packet. if need > len(dest) { @@ -67,64 +55,48 @@ func (b *buffer) fill(need int) error { // if the allocated buffer is not too large, move it to backing storage // to prevent extra allocations on applications that perform large reads if len(dest) <= maxCachedBufSize { - b.dbuf[b.flipcnt&1] = dest + b.cachedBuf = dest } } - // if we're filling the fg buffer, move the existing data to the start of it. - // if we're filling the bg buffer, copy over the data - if n > 0 { - copy(dest[:n], b.buf[b.idx:]) - } - - b.buf = dest - b.idx = 0 + // move the existing data to the start of the buffer. + n := len(b.buf) + copy(dest[:n], b.buf) for { - if b.timeout > 0 { - if err := b.nc.SetReadDeadline(time.Now().Add(b.timeout)); err != nil { - return err - } + nn, err := r(dest[n:]) + n += nn + + if err == nil && n < need { + continue } - nn, err := b.nc.Read(b.buf[n:]) - n += nn + b.buf = dest[:n] - switch err { - case nil: + if err == io.EOF { if n < need { - continue + err = io.ErrUnexpectedEOF + } else { + err = nil } - b.length = n - return nil - - case io.EOF: - if n >= need { - b.length = n - return nil - } - return io.ErrUnexpectedEOF - - default: - return err } + return err } } // returns next N bytes from buffer. // The returned slice is only guaranteed to be valid until the next read -func (b *buffer) readNext(need int) ([]byte, error) { - if b.length < need { +func (b *buffer) readNext(need int, r readerFunc) ([]byte, error) { + if len(b.buf) < need { // refill - if err := b.fill(need); err != nil { + if err := b.fill(need, r); err != nil { return nil, err } } - offset := b.idx - b.idx += need - b.length -= need - return b.buf[offset:b.idx], nil + data := b.buf[:need] + b.buf = b.buf[need:] + return data, nil } // takeBuffer returns a buffer with the requested size. @@ -132,18 +104,18 @@ func (b *buffer) readNext(need int) ([]byte, error) { // Otherwise a bigger buffer is made. // Only one buffer (total) can be used at a time. func (b *buffer) takeBuffer(length int) ([]byte, error) { - if b.length > 0 { + if b.busy() { return nil, ErrBusyBuffer } // test (cheap) general case first - if length <= cap(b.buf) { - return b.buf[:length], nil + if length <= len(b.cachedBuf) { + return b.cachedBuf[:length], nil } - if length < maxPacketSize { - b.buf = make([]byte, length) - return b.buf, nil + if length < maxCachedBufSize { + b.cachedBuf = make([]byte, length) + return b.cachedBuf, nil } // buffer is larger than we want to store. @@ -154,10 +126,10 @@ func (b *buffer) takeBuffer(length int) ([]byte, error) { // known to be smaller than defaultBufSize. // Only one buffer (total) can be used at a time. func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { - if b.length > 0 { + if b.busy() { return nil, ErrBusyBuffer } - return b.buf[:length], nil + return b.cachedBuf[:length], nil } // takeCompleteBuffer returns the complete existing buffer. @@ -165,18 +137,15 @@ func (b *buffer) takeSmallBuffer(length int) ([]byte, error) { // cap and len of the returned buffer will be equal. // Only one buffer (total) can be used at a time. func (b *buffer) takeCompleteBuffer() ([]byte, error) { - if b.length > 0 { + if b.busy() { return nil, ErrBusyBuffer } - return b.buf, nil + return b.cachedBuf, nil } // store stores buf, an updated buffer, if its suitable to do so. -func (b *buffer) store(buf []byte) error { - if b.length > 0 { - return ErrBusyBuffer - } else if cap(buf) <= maxPacketSize && cap(buf) > cap(b.buf) { - b.buf = buf[:cap(buf)] +func (b *buffer) store(buf []byte) { + if cap(buf) <= maxCachedBufSize && cap(buf) > cap(b.cachedBuf) { + b.cachedBuf = buf[:cap(buf)] } - return nil } diff --git a/collations.go b/collations.go index 326a9f7fa..29b1aa43f 100644 --- a/collations.go +++ b/collations.go @@ -8,12 +8,13 @@ package mysql -const defaultCollation = "utf8mb4_general_ci" -const binaryCollation = "binary" +const defaultCollationID = 45 // utf8mb4_general_ci +const binaryCollationID = 63 // A list of available collations mapped to the internal ID. // To update this map use the following MySQL query: -// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID +// +// SELECT COLLATION_NAME, ID FROM information_schema.COLLATIONS WHERE ID<256 ORDER BY ID // // Handshake packet have only 1 byte for collation_id. So we can't use collations with ID > 255. // diff --git a/compress.go b/compress.go new file mode 100644 index 000000000..fa42772ac --- /dev/null +++ b/compress.go @@ -0,0 +1,214 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2024 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "compress/zlib" + "fmt" + "io" + "sync" +) + +var ( + zrPool *sync.Pool // Do not use directly. Use zDecompress() instead. + zwPool *sync.Pool // Do not use directly. Use zCompress() instead. +) + +func init() { + zrPool = &sync.Pool{ + New: func() any { return nil }, + } + zwPool = &sync.Pool{ + New: func() any { + zw, err := zlib.NewWriterLevel(new(bytes.Buffer), 2) + if err != nil { + panic(err) // compress/zlib return non-nil error only if level is invalid + } + return zw + }, + } +} + +func zDecompress(src []byte, dst *bytes.Buffer) (int, error) { + br := bytes.NewReader(src) + var zr io.ReadCloser + var err error + + if a := zrPool.Get(); a == nil { + if zr, err = zlib.NewReader(br); err != nil { + return 0, err + } + } else { + zr = a.(io.ReadCloser) + if err := zr.(zlib.Resetter).Reset(br, nil); err != nil { + return 0, err + } + } + + n, _ := dst.ReadFrom(zr) // ignore err because zr.Close() will return it again. + err = zr.Close() // zr.Close() may return chuecksum error. + zrPool.Put(zr) + return int(n), err +} + +func zCompress(src []byte, dst io.Writer) error { + zw := zwPool.Get().(*zlib.Writer) + zw.Reset(dst) + if _, err := zw.Write(src); err != nil { + return err + } + err := zw.Close() + zwPool.Put(zw) + return err +} + +type compIO struct { + mc *mysqlConn + buff bytes.Buffer +} + +func newCompIO(mc *mysqlConn) *compIO { + return &compIO{ + mc: mc, + } +} + +func (c *compIO) reset() { + c.buff.Reset() +} + +func (c *compIO) readNext(need int, r readerFunc) ([]byte, error) { + for c.buff.Len() < need { + if err := c.readCompressedPacket(r); err != nil { + return nil, err + } + } + data := c.buff.Next(need) + return data[:need:need], nil // prevent caller writes into c.buff +} + +func (c *compIO) readCompressedPacket(r readerFunc) error { + header, err := c.mc.buf.readNext(7, r) // size of compressed header + if err != nil { + return err + } + _ = header[6] // bounds check hint to compiler; guaranteed by readNext + + // compressed header structure + comprLength := getUint24(header[0:3]) + compressionSequence := uint8(header[3]) + uncompressedLength := getUint24(header[4:7]) + if debug { + fmt.Printf("uncompress cmplen=%v uncomplen=%v pkt_cmp_seq=%v expected_cmp_seq=%v\n", + comprLength, uncompressedLength, compressionSequence, c.mc.sequence) + } + // Do not return ErrPktSync here. + // Server may return error packet (e.g. 1153 Got a packet bigger than 'max_allowed_packet' bytes) + // before receiving all packets from client. In this case, seqnr is younger than expected. + // NOTE: Both of mariadbclient and mysqlclient do not check seqnr. Only server checks it. + if debug && compressionSequence != c.mc.sequence { + fmt.Printf("WARN: unexpected cmpress seq nr: expected %v, got %v", + c.mc.sequence, compressionSequence) + } + c.mc.sequence = compressionSequence + 1 + c.mc.compressSequence = c.mc.sequence + + comprData, err := c.mc.buf.readNext(comprLength, r) + if err != nil { + return err + } + + // if payload is uncompressed, its length will be specified as zero, and its + // true length is contained in comprLength + if uncompressedLength == 0 { + c.buff.Write(comprData) + return nil + } + + // use existing capacity in bytesBuf if possible + c.buff.Grow(uncompressedLength) + nread, err := zDecompress(comprData, &c.buff) + if err != nil { + return err + } + if nread != uncompressedLength { + return fmt.Errorf("invalid compressed packet: uncompressed length in header is %d, actual %d", + uncompressedLength, nread) + } + return nil +} + +const minCompressLength = 150 +const maxPayloadLen = maxPacketSize - 4 + +// writePackets sends one or some packets with compression. +// Use this instead of mc.netConn.Write() when mc.compress is true. +func (c *compIO) writePackets(packets []byte) (int, error) { + totalBytes := len(packets) + blankHeader := make([]byte, 7) + buf := &c.buff + + for len(packets) > 0 { + payloadLen := min(maxPayloadLen, len(packets)) + payload := packets[:payloadLen] + uncompressedLen := payloadLen + + buf.Reset() + buf.Write(blankHeader) // Buffer.Write() never returns error + + // If payload is less than minCompressLength, don't compress. + if uncompressedLen < minCompressLength { + buf.Write(payload) + uncompressedLen = 0 + } else { + err := zCompress(payload, buf) + if debug && err != nil { + fmt.Printf("zCompress error: %v", err) + } + // do not compress if compressed data is larger than uncompressed data + // I intentionally miss 7 byte header in the buf; zCompress must compress more than 7 bytes. + if err != nil || buf.Len() >= uncompressedLen { + buf.Reset() + buf.Write(blankHeader) + buf.Write(payload) + uncompressedLen = 0 + } + } + + if n, err := c.writeCompressedPacket(buf.Bytes(), uncompressedLen); err != nil { + // To allow returning ErrBadConn when sending really 0 bytes, we sum + // up compressed bytes that is returned by underlying Write(). + return totalBytes - len(packets) + n, err + } + packets = packets[payloadLen:] + } + + return totalBytes, nil +} + +// writeCompressedPacket writes a compressed packet with header. +// data should start with 7 size space for header followed by payload. +func (c *compIO) writeCompressedPacket(data []byte, uncompressedLen int) (int, error) { + mc := c.mc + comprLength := len(data) - 7 + if debug { + fmt.Printf( + "writeCompressedPacket: comprLength=%v, uncompressedLen=%v, seq=%v", + comprLength, uncompressedLen, mc.compressSequence) + } + + // compression header + putUint24(data[0:3], comprLength) + data[3] = mc.compressSequence + putUint24(data[4:7], uncompressedLen) + + mc.compressSequence++ + return mc.writeWithTimeout(data) +} diff --git a/compress_test.go b/compress_test.go new file mode 100644 index 000000000..030deaefa --- /dev/null +++ b/compress_test.go @@ -0,0 +1,119 @@ +// Go MySQL Driver - A MySQL-Driver for Go's database/sql package +// +// Copyright 2024 The Go-MySQL-Driver Authors. All rights reserved. +// +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this file, +// You can obtain one at http://mozilla.org/MPL/2.0/. + +package mysql + +import ( + "bytes" + "crypto/rand" + "io" + "testing" +) + +func makeRandByteSlice(size int) []byte { + randBytes := make([]byte, size) + rand.Read(randBytes) + return randBytes +} + +// compressHelper compresses uncompressedPacket and checks state variables +func compressHelper(t *testing.T, mc *mysqlConn, uncompressedPacket []byte) []byte { + conn := new(mockConn) + mc.netConn = conn + + err := mc.writePacket(append(make([]byte, 4), uncompressedPacket...)) + if err != nil { + t.Fatal(err) + } + + return conn.written +} + +// uncompressHelper uncompresses compressedPacket and checks state variables +func uncompressHelper(t *testing.T, mc *mysqlConn, compressedPacket []byte) []byte { + // mocking out buf variable + conn := new(mockConn) + conn.data = compressedPacket + mc.netConn = conn + + uncompressedPacket, err := mc.readPacket() + if err != nil { + if err != io.EOF { + t.Fatalf("non-nil/non-EOF error when reading contents: %s", err.Error()) + } + } + return uncompressedPacket +} + +// roundtripHelper compresses then uncompresses uncompressedPacket and checks state variables +func roundtripHelper(t *testing.T, cSend *mysqlConn, cReceive *mysqlConn, uncompressedPacket []byte) []byte { + compressed := compressHelper(t, cSend, uncompressedPacket) + return uncompressHelper(t, cReceive, compressed) +} + +// TestRoundtrip tests two connections, where one is reading and the other is writing +func TestRoundtrip(t *testing.T) { + tests := []struct { + uncompressed []byte + desc string + }{ + {uncompressed: []byte("a"), + desc: "a"}, + {uncompressed: []byte("hello world"), + desc: "hello world"}, + {uncompressed: make([]byte, 100), + desc: "100 bytes"}, + {uncompressed: make([]byte, 32768), + desc: "32768 bytes"}, + {uncompressed: make([]byte, 330000), + desc: "33000 bytes"}, + {uncompressed: makeRandByteSlice(10), + desc: "10 rand bytes", + }, + {uncompressed: makeRandByteSlice(100), + desc: "100 rand bytes", + }, + {uncompressed: makeRandByteSlice(32768), + desc: "32768 rand bytes", + }, + {uncompressed: bytes.Repeat(makeRandByteSlice(100), 10000), + desc: "100 rand * 10000 repeat bytes", + }, + } + + _, cSend := newRWMockConn(0) + cSend.compress = true + cSend.compIO = newCompIO(cSend) + _, cReceive := newRWMockConn(0) + cReceive.compress = true + cReceive.compIO = newCompIO(cReceive) + + for _, test := range tests { + t.Run(test.desc, func(t *testing.T) { + cSend.resetSequence() + cReceive.resetSequence() + + uncompressed := roundtripHelper(t, cSend, cReceive, test.uncompressed) + if len(uncompressed) != len(test.uncompressed) { + t.Errorf("uncompressed size is unexpected. expected %d but got %d", + len(test.uncompressed), len(uncompressed)) + } + if !bytes.Equal(uncompressed, test.uncompressed) { + t.Errorf("roundtrip failed") + } + if cSend.sequence != cReceive.sequence { + t.Errorf("inconsistent sequence number: send=%v recv=%v", + cSend.sequence, cReceive.sequence) + } + if cSend.compressSequence != cReceive.compressSequence { + t.Errorf("inconsistent compress sequence number: send=%v recv=%v", + cSend.compressSequence, cReceive.compressSequence) + } + }) + } +} diff --git a/conncheck.go b/conncheck.go index 024eb2858..0ea721720 100644 --- a/conncheck.go +++ b/conncheck.go @@ -6,6 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. +//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd || solaris || illumos // +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql diff --git a/conncheck_dummy.go b/conncheck_dummy.go index ea7fb607a..a56c138f2 100644 --- a/conncheck_dummy.go +++ b/conncheck_dummy.go @@ -6,6 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. +//go:build !linux && !darwin && !dragonfly && !freebsd && !netbsd && !openbsd && !solaris && !illumos // +build !linux,!darwin,!dragonfly,!freebsd,!netbsd,!openbsd,!solaris,!illumos package mysql diff --git a/conncheck_test.go b/conncheck_test.go index 53995517b..6b60cb7d6 100644 --- a/conncheck_test.go +++ b/conncheck_test.go @@ -6,6 +6,7 @@ // License, v. 2.0. If a copy of the MPL was not distributed with this file, // You can obtain one at http://mozilla.org/MPL/2.0/. +//go:build linux || darwin || dragonfly || freebsd || netbsd || openbsd || solaris || illumos // +build linux darwin dragonfly freebsd netbsd openbsd solaris illumos package mysql @@ -16,7 +17,7 @@ import ( ) func TestStaleConnectionChecks(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { dbt.mustExec("SET @@SESSION.wait_timeout = 2") if err := dbt.db.Ping(); err != nil { diff --git a/connection.go b/connection.go index 835f89729..3e455a3ff 100644 --- a/connection.go +++ b/connection.go @@ -13,28 +13,32 @@ import ( "database/sql" "database/sql/driver" "encoding/json" + "fmt" "io" "net" + "runtime" "strconv" "strings" + "sync/atomic" "time" ) type mysqlConn struct { buf buffer netConn net.Conn - rawConn net.Conn // underlying connection when netConn is TLS connection. - affectedRows uint64 - insertId uint64 + rawConn net.Conn // underlying connection when netConn is TLS connection. + result mysqlResult // managed by clearResult() and handleOkPacket(). + compIO *compIO cfg *Config + connector *connector maxAllowedPacket int maxWriteSize int - writeTimeout time.Duration flags clientFlag status statusFlag sequence uint8 + compressSequence uint8 parseTime bool - reset bool // set when the Go SQL package calls ResetSession + compress bool // for context support (Go 1.8+) watching bool @@ -42,61 +46,92 @@ type mysqlConn struct { closech chan struct{} finished chan<- struct{} canceled atomicError // set non-nil if conn is canceled - closed atomicBool // set when conn is closed, before closech is closed + closed atomic.Bool // set when conn is closed, before closech is closed +} + +// Helper function to call per-connection logger. +func (mc *mysqlConn) log(v ...any) { + _, filename, lineno, ok := runtime.Caller(1) + if ok { + pos := strings.LastIndexByte(filename, '/') + if pos != -1 { + filename = filename[pos+1:] + } + prefix := fmt.Sprintf("%s:%d ", filename, lineno) + v = append([]any{prefix}, v...) + } + + mc.cfg.Logger.Print(v...) +} + +func (mc *mysqlConn) readWithTimeout(b []byte) (int, error) { + to := mc.cfg.ReadTimeout + if to > 0 { + if err := mc.netConn.SetReadDeadline(time.Now().Add(to)); err != nil { + return 0, err + } + } + return mc.netConn.Read(b) +} + +func (mc *mysqlConn) writeWithTimeout(b []byte) (int, error) { + to := mc.cfg.WriteTimeout + if to > 0 { + if err := mc.netConn.SetWriteDeadline(time.Now().Add(to)); err != nil { + return 0, err + } + } + return mc.netConn.Write(b) +} + +func (mc *mysqlConn) resetSequence() { + mc.sequence = 0 + mc.compressSequence = 0 +} + +// syncSequence must be called when finished writing some packet and before start reading. +func (mc *mysqlConn) syncSequence() { + // Syncs compressionSequence to sequence. + // This is not documented but done in `net_flush()` in MySQL and MariaDB. + // https://github.com/mariadb-corporation/mariadb-connector-c/blob/8228164f850b12353da24df1b93a1e53cc5e85e9/libmariadb/ma_net.c#L170-L171 + // https://github.com/mysql/mysql-server/blob/824e2b4064053f7daf17d7f3f84b7a3ed92e5fb4/sql-common/net_serv.cc#L293 + if mc.compress { + mc.sequence = mc.compressSequence + mc.compIO.reset() + } } // Handles parameters set in DSN after the connection is established func (mc *mysqlConn) handleParams() (err error) { var cmdSet strings.Builder - for param, val := range mc.cfg.Params { - switch param { - // Charset: character_set_connection, character_set_client, character_set_results - case "charset": - charsets := strings.Split(val, ",") - for i := range charsets { - // ignore errors here - a charset may not exist - err = mc.exec("SET NAMES " + charsets[i]) - if err == nil { - break - } - } - if err != nil { - return - } - // Other system vars accumulated in a single SET command - default: - if cmdSet.Len() == 0 { - // Heuristic: 29 chars for each other key=value to reduce reallocations - cmdSet.Grow(4 + len(param) + 1 + len(val) + 30*(len(mc.cfg.Params)-1)) - cmdSet.WriteString("SET ") - } else { - cmdSet.WriteByte(',') - } - cmdSet.WriteString(param) - cmdSet.WriteByte('=') - cmdSet.WriteString(val) + for param, val := range mc.cfg.Params { + if cmdSet.Len() == 0 { + // Heuristic: 29 chars for each other key=value to reduce reallocations + cmdSet.Grow(4 + len(param) + 3 + len(val) + 30*(len(mc.cfg.Params)-1)) + cmdSet.WriteString("SET ") + } else { + cmdSet.WriteString(", ") } + cmdSet.WriteString(param) + cmdSet.WriteString(" = ") + cmdSet.WriteString(val) } if cmdSet.Len() > 0 { err = mc.exec(cmdSet.String()) - if err != nil { - return - } } return } +// markBadConn replaces errBadConnNoWrite with driver.ErrBadConn. +// This function is used to return driver.ErrBadConn only when safe to retry. func (mc *mysqlConn) markBadConn(err error) error { - if mc == nil { - return err - } - if err != errBadConnNoWrite { - return err + if err == errBadConnNoWrite { + return driver.ErrBadConn } - return driver.ErrBadConn + return err } func (mc *mysqlConn) Begin() (driver.Tx, error) { @@ -104,8 +139,7 @@ func (mc *mysqlConn) Begin() (driver.Tx, error) { } func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) { - if mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + if mc.closed.Load() { return nil, driver.ErrBadConn } var q string @@ -123,13 +157,17 @@ func (mc *mysqlConn) begin(readOnly bool) (driver.Tx, error) { func (mc *mysqlConn) Close() (err error) { // Makes Close idempotent - if !mc.closed.IsSet() { + if !mc.closed.Load() { err = mc.writeCommandPacket(comQuit) } + mc.close() + return +} +// close closes the network connection and clear results without sending COM_QUIT. +func (mc *mysqlConn) close() { mc.cleanup() - - return + mc.clearResult() } // Closes the network connection and unsets internal variables. Do not call this @@ -137,22 +175,26 @@ func (mc *mysqlConn) Close() (err error) { // is called before auth or on auth failure because MySQL will have already // closed the network connection. func (mc *mysqlConn) cleanup() { - if !mc.closed.TrySet(true) { + if mc.closed.Swap(true) { return } // Makes cleanup idempotent close(mc.closech) - if mc.netConn == nil { + conn := mc.rawConn + if conn == nil { return } - if err := mc.netConn.Close(); err != nil { - errLog.Print(err) + if err := conn.Close(); err != nil { + mc.log("closing connection:", err) } + // This function can be called from multiple goroutines. + // So we can not mc.clearResult() here. + // Caller should do it if they are in safe goroutine. } func (mc *mysqlConn) error() error { - if mc.closed.IsSet() { + if mc.closed.Load() { if err := mc.canceled.Value(); err != nil { return err } @@ -162,15 +204,14 @@ func (mc *mysqlConn) error() error { } func (mc *mysqlConn) Prepare(query string) (driver.Stmt, error) { - if mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + if mc.closed.Load() { return nil, driver.ErrBadConn } // Send command err := mc.writeCommandPacketStr(comStmtPrepare, query) if err != nil { // STMT_PREPARE is safe to retry. So we can return ErrBadConn here. - errLog.Print(err) + mc.log(err) return nil, driver.ErrBadConn } @@ -204,8 +245,10 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin buf, err := mc.buf.takeCompleteBuffer() if err != nil { // can not take the buffer. Something must be wrong with the connection - errLog.Print(err) - return "", ErrInvalidConn + mc.cleanup() + // interpolateParams would be called before sending any query. + // So its safe to retry. + return "", driver.ErrBadConn } buf = buf[:0] argPos := 0 @@ -246,7 +289,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin buf = append(buf, "'0000-00-00'"...) } else { buf = append(buf, '\'') - buf, err = appendDateTime(buf, v.In(mc.cfg.Loc)) + buf, err = appendDateTime(buf, v.In(mc.cfg.Loc), mc.cfg.timeTruncate) if err != nil { return "", err } @@ -295,8 +338,7 @@ func (mc *mysqlConn) interpolateParams(query string, args []driver.Value) (strin } func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, error) { - if mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + if mc.closed.Load() { return nil, driver.ErrBadConn } if len(args) != 0 { @@ -310,28 +352,25 @@ func (mc *mysqlConn) Exec(query string, args []driver.Value) (driver.Result, err } query = prepared } - mc.affectedRows = 0 - mc.insertId = 0 err := mc.exec(query) if err == nil { - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, err + copied := mc.result + return &copied, err } return nil, mc.markBadConn(err) } // Internal function to execute commands func (mc *mysqlConn) exec(query string) error { + handleOk := mc.clearResult() // Send command if err := mc.writeCommandPacketStr(comQuery, query); err != nil { return mc.markBadConn(err) } // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return err } @@ -348,7 +387,7 @@ func (mc *mysqlConn) exec(query string) error { } } - return mc.discardResults() + return handleOk.discardResults() } func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, error) { @@ -356,8 +395,9 @@ func (mc *mysqlConn) Query(query string, args []driver.Value) (driver.Rows, erro } func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) { - if mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + handleOk := mc.clearResult() + + if mc.closed.Load() { return nil, driver.ErrBadConn } if len(args) != 0 { @@ -373,43 +413,47 @@ func (mc *mysqlConn) query(query string, args []driver.Value) (*textRows, error) } // Send command err := mc.writeCommandPacketStr(comQuery, query) - if err == nil { - // Read Result - var resLen int - resLen, err = mc.readResultSetHeaderPacket() - if err == nil { - rows := new(textRows) - rows.mc = mc + if err != nil { + return nil, mc.markBadConn(err) + } - if resLen == 0 { - rows.rs.done = true + // Read Result + var resLen int + resLen, err = handleOk.readResultSetHeaderPacket() + if err != nil { + return nil, err + } - switch err := rows.NextResultSet(); err { - case nil, io.EOF: - return rows, nil - default: - return nil, err - } - } + rows := new(textRows) + rows.mc = mc - // Columns - rows.rs.columns, err = mc.readColumns(resLen) - return rows, err + if resLen == 0 { + rows.rs.done = true + + switch err := rows.NextResultSet(); err { + case nil, io.EOF: + return rows, nil + default: + return nil, err } } - return nil, mc.markBadConn(err) + + // Columns + rows.rs.columns, err = mc.readColumns(resLen) + return rows, err } // Gets the value of the given MySQL System Variable // The returned byte slice is only valid until the next read func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { // Send command + handleOk := mc.clearResult() if err := mc.writeCommandPacketStr(comQuery, "SELECT @@"+name); err != nil { return nil, err } // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err == nil { rows := new(textRows) rows.mc = mc @@ -430,7 +474,7 @@ func (mc *mysqlConn) getSystemVar(name string) ([]byte, error) { return nil, err } -// finish is called when the query has canceled. +// cancel is called when the query has canceled. func (mc *mysqlConn) cancel(err error) { mc.canceled.Set(err) mc.cleanup() @@ -450,8 +494,7 @@ func (mc *mysqlConn) finish() { // Ping implements driver.Pinger interface func (mc *mysqlConn) Ping(ctx context.Context) (err error) { - if mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + if mc.closed.Load() { return driver.ErrBadConn } @@ -460,16 +503,17 @@ func (mc *mysqlConn) Ping(ctx context.Context) (err error) { } defer mc.finish() + handleOk := mc.clearResult() if err = mc.writeCommandPacket(comPing); err != nil { return mc.markBadConn(err) } - return mc.readResultOK() + return handleOk.readResultOK() } // BeginTx implements driver.ConnBeginTx interface func (mc *mysqlConn) BeginTx(ctx context.Context, opts driver.TxOptions) (driver.Tx, error) { - if mc.closed.IsSet() { + if mc.closed.Load() { return nil, driver.ErrBadConn } @@ -636,15 +680,42 @@ func (mc *mysqlConn) CheckNamedValue(nv *driver.NamedValue) (err error) { // ResetSession implements driver.SessionResetter. // (From Go 1.10) func (mc *mysqlConn) ResetSession(ctx context.Context) error { - if mc.closed.IsSet() { + if mc.closed.Load() || mc.buf.busy() { return driver.ErrBadConn } - mc.reset = true + + // Perform a stale connection check. We only perform this check for + // the first query on a connection that has been checked out of the + // connection pool: a fresh connection from the pool is more likely + // to be stale, and it has not performed any previous writes that + // could cause data corruption, so it's safe to return ErrBadConn + // if the check fails. + if mc.cfg.CheckConnLiveness { + conn := mc.netConn + if mc.rawConn != nil { + conn = mc.rawConn + } + var err error + if mc.cfg.ReadTimeout != 0 { + err = conn.SetReadDeadline(time.Now().Add(mc.cfg.ReadTimeout)) + } + if err == nil { + err = connCheck(conn) + } + if err != nil { + mc.log("closing bad idle connection: ", err) + return driver.ErrBadConn + } + } + return nil } // IsValid implements driver.Validator interface // (From Go 1.15) func (mc *mysqlConn) IsValid() bool { - return !mc.closed.IsSet() + return !mc.closed.Load() && !mc.buf.busy() } + +var _ driver.SessionResetter = &mysqlConn{} +var _ driver.Validator = &mysqlConn{} diff --git a/connection_test.go b/connection_test.go index a6d677308..f7740898e 100644 --- a/connection_test.go +++ b/connection_test.go @@ -19,7 +19,7 @@ import ( func TestInterpolateParams(t *testing.T) { mc := &mysqlConn{ - buf: newBuffer(nil), + buf: newBuffer(), maxAllowedPacket: maxPacketSize, cfg: &Config{ InterpolateParams: true, @@ -39,7 +39,7 @@ func TestInterpolateParams(t *testing.T) { func TestInterpolateParamsJSONRawMessage(t *testing.T) { mc := &mysqlConn{ - buf: newBuffer(nil), + buf: newBuffer(), maxAllowedPacket: maxPacketSize, cfg: &Config{ InterpolateParams: true, @@ -66,7 +66,7 @@ func TestInterpolateParamsJSONRawMessage(t *testing.T) { func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { mc := &mysqlConn{ - buf: newBuffer(nil), + buf: newBuffer(), maxAllowedPacket: maxPacketSize, cfg: &Config{ InterpolateParams: true, @@ -83,7 +83,7 @@ func TestInterpolateParamsTooManyPlaceholders(t *testing.T) { // https://github.com/go-sql-driver/mysql/pull/490 func TestInterpolateParamsPlaceholderInString(t *testing.T) { mc := &mysqlConn{ - buf: newBuffer(nil), + buf: newBuffer(), maxAllowedPacket: maxPacketSize, cfg: &Config{ InterpolateParams: true, @@ -99,7 +99,7 @@ func TestInterpolateParamsPlaceholderInString(t *testing.T) { func TestInterpolateParamsUint64(t *testing.T) { mc := &mysqlConn{ - buf: newBuffer(nil), + buf: newBuffer(), maxAllowedPacket: maxPacketSize, cfg: &Config{ InterpolateParams: true, @@ -117,8 +117,8 @@ func TestInterpolateParamsUint64(t *testing.T) { func TestCheckNamedValue(t *testing.T) { value := driver.NamedValue{Value: ^uint64(0)} - x := &mysqlConn{} - err := x.CheckNamedValue(&value) + mc := &mysqlConn{} + err := mc.CheckNamedValue(&value) if err != nil { t.Fatal("uint64 high-bit not convertible", err) @@ -147,7 +147,7 @@ func TestCleanCancel(t *testing.T) { t.Errorf("expected context.Canceled, got %#v", err) } - if mc.closed.IsSet() { + if mc.closed.Load() { t.Error("expected mc is not closed, closed actually") } @@ -159,13 +159,15 @@ func TestCleanCancel(t *testing.T) { func TestPingMarkBadConnection(t *testing.T) { nc := badConnection{err: errors.New("boom")} - ms := &mysqlConn{ + mc := &mysqlConn{ netConn: nc, - buf: newBuffer(nc), + buf: newBuffer(), maxAllowedPacket: defaultMaxAllowedPacket, + closech: make(chan struct{}), + cfg: NewConfig(), } - err := ms.Ping(context.Background()) + err := mc.Ping(context.Background()) if err != driver.ErrBadConn { t.Errorf("expected driver.ErrBadConn, got %#v", err) @@ -174,17 +176,18 @@ func TestPingMarkBadConnection(t *testing.T) { func TestPingErrInvalidConn(t *testing.T) { nc := badConnection{err: errors.New("failed to write"), n: 10} - ms := &mysqlConn{ + mc := &mysqlConn{ netConn: nc, - buf: newBuffer(nc), + buf: newBuffer(), maxAllowedPacket: defaultMaxAllowedPacket, closech: make(chan struct{}), + cfg: NewConfig(), } - err := ms.Ping(context.Background()) + err := mc.Ping(context.Background()) - if err != ErrInvalidConn { - t.Errorf("expected ErrInvalidConn, got %#v", err) + if err != nc.err { + t.Errorf("expected %#v, got %#v", nc.err, err) } } diff --git a/connector.go b/connector.go index d567b4e4f..bc1d46afc 100644 --- a/connector.go +++ b/connector.go @@ -11,11 +11,55 @@ package mysql import ( "context" "database/sql/driver" + "fmt" "net" + "os" + "strconv" + "strings" ) type connector struct { - cfg *Config // immutable private copy. + cfg *Config // immutable private copy. + encodedAttributes string // Encoded connection attributes. +} + +func encodeConnectionAttributes(cfg *Config) string { + connAttrsBuf := make([]byte, 0) + + // default connection attributes + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrClientName) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrClientNameValue) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrOS) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrOSValue) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPlatform) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPlatformValue) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrPid) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, strconv.Itoa(os.Getpid())) + serverHost, _, _ := net.SplitHostPort(cfg.Addr) + if serverHost != "" { + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, connAttrServerHost) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, serverHost) + } + + // user-defined connection attributes + for _, connAttr := range strings.Split(cfg.ConnectionAttributes, ",") { + k, v, found := strings.Cut(connAttr, ":") + if !found { + continue + } + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, k) + connAttrsBuf = appendLengthEncodedString(connAttrsBuf, v) + } + + return string(connAttrsBuf) +} + +func newConnector(cfg *Config) *connector { + encodedAttributes := encodeConnectionAttributes(cfg) + return &connector{ + cfg: cfg, + encodedAttributes: encodedAttributes, + } } // Connect implements driver.Connector interface. @@ -23,43 +67,56 @@ type connector struct { func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { var err error + // Invoke beforeConnect if present, with a copy of the configuration + cfg := c.cfg + if c.cfg.beforeConnect != nil { + cfg = c.cfg.Clone() + err = c.cfg.beforeConnect(ctx, cfg) + if err != nil { + return nil, err + } + } + // New mysqlConn mc := &mysqlConn{ maxAllowedPacket: maxPacketSize, maxWriteSize: maxPacketSize - 1, closech: make(chan struct{}), - cfg: c.cfg, + cfg: cfg, + connector: c, } mc.parseTime = mc.cfg.ParseTime // Connect to Server - dialsLock.RLock() - dial, ok := dials[mc.cfg.Net] - dialsLock.RUnlock() - if ok { - dctx := ctx - if mc.cfg.Timeout > 0 { - var cancel context.CancelFunc - dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) - defer cancel() - } - mc.netConn, err = dial(dctx, mc.cfg.Addr) - } else { - nd := net.Dialer{Timeout: mc.cfg.Timeout} - mc.netConn, err = nd.DialContext(ctx, mc.cfg.Net, mc.cfg.Addr) + dctx := ctx + if mc.cfg.Timeout > 0 { + var cancel context.CancelFunc + dctx, cancel = context.WithTimeout(ctx, c.cfg.Timeout) + defer cancel() } + if c.cfg.DialFunc != nil { + mc.netConn, err = c.cfg.DialFunc(dctx, mc.cfg.Net, mc.cfg.Addr) + } else { + dialsLock.RLock() + dial, ok := dials[mc.cfg.Net] + dialsLock.RUnlock() + if ok { + mc.netConn, err = dial(dctx, mc.cfg.Addr) + } else { + nd := net.Dialer{} + mc.netConn, err = nd.DialContext(dctx, mc.cfg.Net, mc.cfg.Addr) + } + } if err != nil { return nil, err } + mc.rawConn = mc.netConn // Enable TCP Keepalives on TCP connections if tc, ok := mc.netConn.(*net.TCPConn); ok { if err := tc.SetKeepAlive(true); err != nil { - // Don't send COM_QUIT before handshake. - mc.netConn.Close() - mc.netConn = nil - return nil, err + c.cfg.Logger.Print(err) } } @@ -71,11 +128,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { } defer mc.finish() - mc.buf = newBuffer(mc.netConn) - - // Set I/O timeouts - mc.buf.timeout = mc.cfg.ReadTimeout - mc.writeTimeout = mc.cfg.WriteTimeout + mc.buf = newBuffer() // Reading Handshake Initialization Packet authData, plugin, err := mc.readHandshakePacket() @@ -92,7 +145,7 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { authResp, err := mc.auth(authData, plugin) if err != nil { // try the default auth plugin, if using the requested plugin failed - errLog.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) + c.cfg.Logger.Print("could not use requested auth plugin '"+plugin+"': ", err.Error()) plugin = defaultAuthPlugin authResp, err = mc.auth(authData, plugin) if err != nil { @@ -114,6 +167,10 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { return nil, err } + if mc.cfg.compress && mc.flags&clientCompress == clientCompress { + mc.compress = true + mc.compIO = newCompIO(mc) + } if mc.cfg.MaxAllowedPacket > 0 { mc.maxAllowedPacket = mc.cfg.MaxAllowedPacket } else { @@ -123,12 +180,36 @@ func (c *connector) Connect(ctx context.Context) (driver.Conn, error) { mc.Close() return nil, err } - mc.maxAllowedPacket = stringToInt(maxap) - 1 + n, err := strconv.Atoi(string(maxap)) + if err != nil { + mc.Close() + return nil, fmt.Errorf("invalid max_allowed_packet value (%q): %w", maxap, err) + } + mc.maxAllowedPacket = n - 1 } if mc.maxAllowedPacket < maxPacketSize { mc.maxWriteSize = mc.maxAllowedPacket } + // Charset: character_set_connection, character_set_client, character_set_results + if len(mc.cfg.charsets) > 0 { + for _, cs := range mc.cfg.charsets { + // ignore errors here - a charset may not exist + if mc.cfg.Collation != "" { + err = mc.exec("SET NAMES " + cs + " COLLATE " + mc.cfg.Collation) + } else { + err = mc.exec("SET NAMES " + cs) + } + if err == nil { + break + } + } + if err != nil { + mc.Close() + return nil, err + } + } + // Handle DSN Params err = mc.handleParams() if err != nil { diff --git a/connector_test.go b/connector_test.go index 976903c5b..82d8c5989 100644 --- a/connector_test.go +++ b/connector_test.go @@ -8,11 +8,11 @@ import ( ) func TestConnectorReturnsTimeout(t *testing.T) { - connector := &connector{&Config{ + connector := newConnector(&Config{ Net: "tcp", Addr: "1.1.1.1:1234", Timeout: 10 * time.Millisecond, - }} + }) _, err := connector.Connect(context.Background()) if err == nil { diff --git a/const.go b/const.go index b1e6b85ef..4aadcd642 100644 --- a/const.go +++ b/const.go @@ -8,12 +8,27 @@ package mysql +import "runtime" + const ( + debug = false // for debugging. Set true only in development. + defaultAuthPlugin = "mysql_native_password" - defaultMaxAllowedPacket = 4 << 20 // 4 MiB + defaultMaxAllowedPacket = 64 << 20 // 64 MiB. See https://github.com/go-sql-driver/mysql/issues/1355 minProtocolVersion = 10 maxPacketSize = 1<<24 - 1 timeFormat = "2006-01-02 15:04:05.999999" + + // Connection attributes + // See https://dev.mysql.com/doc/refman/8.0/en/performance-schema-connection-attribute-tables.html#performance-schema-connection-attributes-available + connAttrClientName = "_client_name" + connAttrClientNameValue = "Go-MySQL-Driver" + connAttrOS = "_os" + connAttrOSValue = runtime.GOOS + connAttrPlatform = "_platform" + connAttrPlatformValue = runtime.GOARCH + connAttrPid = "_pid" + connAttrServerHost = "_server_host" ) // MySQL constants documentation: @@ -112,7 +127,10 @@ const ( fieldTypeBit ) const ( - fieldTypeJSON fieldType = iota + 0xf5 + fieldTypeVector fieldType = iota + 0xf2 + fieldTypeInvalid + fieldTypeBool + fieldTypeJSON fieldTypeNewDecimal fieldTypeEnum fieldTypeSet diff --git a/driver.go b/driver.go index c1bdf1199..105316b81 100644 --- a/driver.go +++ b/driver.go @@ -8,10 +8,10 @@ // // The driver should be used via the database/sql package: // -// import "database/sql" -// import _ "github.com/go-sql-driver/mysql" +// import "database/sql" +// import _ "github.com/go-sql-driver/mysql" // -// db, err := sql.Open("mysql", "user:password@/dbname") +// db, err := sql.Open("mysql", "user:password@/dbname") // // See https://github.com/go-sql-driver/mysql#usage for details package mysql @@ -55,6 +55,15 @@ func RegisterDialContext(net string, dial DialContextFunc) { dials[net] = dial } +// DeregisterDialContext removes the custom dial function registered with the given net. +func DeregisterDialContext(net string) { + dialsLock.Lock() + defer dialsLock.Unlock() + if dials != nil { + delete(dials, net) + } +} + // RegisterDial registers a custom dial function. It can then be used by the // network address mynet(addr), where mynet is the registered new network. // addr is passed as a parameter to the dial function. @@ -74,14 +83,18 @@ func (d MySQLDriver) Open(dsn string) (driver.Conn, error) { if err != nil { return nil, err } - c := &connector{ - cfg: cfg, - } + c := newConnector(cfg) return c.Connect(context.Background()) } +// This variable can be replaced with -ldflags like below: +// go build "-ldflags=-X github.com/go-sql-driver/mysql.driverName=custom" +var driverName = "mysql" + func init() { - sql.Register("mysql", &MySQLDriver{}) + if driverName != "" { + sql.Register(driverName, &MySQLDriver{}) + } } // NewConnector returns new driver.Connector. @@ -92,7 +105,7 @@ func NewConnector(cfg *Config) (driver.Connector, error) { if err := cfg.normalize(); err != nil { return nil, err } - return &connector{cfg: cfg}, nil + return newConnector(cfg), nil } // OpenConnector implements driver.DriverContext. @@ -101,7 +114,5 @@ func (d MySQLDriver) OpenConnector(dsn string) (driver.Connector, error) { if err != nil { return nil, err } - return &connector{ - cfg: cfg, - }, nil + return newConnector(cfg), nil } diff --git a/driver_test.go b/driver_test.go index 4850498d0..00e828657 100644 --- a/driver_test.go +++ b/driver_test.go @@ -11,20 +11,22 @@ package mysql import ( "bytes" "context" + "crypto/rand" "crypto/tls" "database/sql" "database/sql/driver" "encoding/json" "fmt" "io" - "io/ioutil" "log" "math" + mrand "math/rand" "net" "net/url" "os" "reflect" "runtime" + "strconv" "strings" "sync" "sync/atomic" @@ -32,6 +34,16 @@ import ( "time" ) +// This variable can be replaced with -ldflags like below: +// go test "-ldflags=-X github.com/go-sql-driver/mysql.driverNameTest=custom" +var driverNameTest string + +func init() { + if driverNameTest == "" { + driverNameTest = driverName + } +} + // Ensure that all the driver interfaces are implemented var ( _ driver.Rows = &binaryRows{} @@ -83,7 +95,7 @@ func init() { } type DBTest struct { - *testing.T + testing.TB db *sql.DB } @@ -112,12 +124,14 @@ func runTestsWithMultiStatement(t *testing.T, dsn string, tests ...func(dbt *DBT dsn += "&multiStatements=true" var db *sql.DB if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { - db, err = sql.Open("mysql", dsn) + db, err = sql.Open(driverNameTest, dsn) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } defer db.Close() } + // Previous test may be skipped without dropping the test table + db.Exec("DROP TABLE IF EXISTS test") dbt := &DBTest{t, db} for _, test := range tests { @@ -131,59 +145,131 @@ func runTests(t *testing.T, dsn string, tests ...func(dbt *DBTest)) { t.Skipf("MySQL server not running on %s", netAddr) } - db, err := sql.Open("mysql", dsn) + db, err := sql.Open(driverNameTest, dsn) if err != nil { - t.Fatalf("error connecting: %s", err.Error()) + t.Fatalf("connecting %q: %s", dsn, err) } defer db.Close() - - db.Exec("DROP TABLE IF EXISTS test") + if err = db.Ping(); err != nil { + t.Fatalf("connecting %q: %s", dsn, err) + } dsn2 := dsn + "&interpolateParams=true" var db2 *sql.DB if _, err := ParseDSN(dsn2); err != errInvalidDSNUnsafeCollation { - db2, err = sql.Open("mysql", dsn2) + db2, err = sql.Open(driverNameTest, dsn2) if err != nil { - t.Fatalf("error connecting: %s", err.Error()) + t.Fatalf("connecting %q: %s", dsn2, err) } defer db2.Close() } - dsn3 := dsn + "&multiStatements=true" + dsn3 := dsn + "&compress=true" var db3 *sql.DB - if _, err := ParseDSN(dsn3); err != errInvalidDSNUnsafeCollation { - db3, err = sql.Open("mysql", dsn3) - if err != nil { - t.Fatalf("error connecting: %s", err.Error()) - } - defer db3.Close() + db3, err = sql.Open(driverNameTest, dsn3) + if err != nil { + t.Fatalf("connecting %q: %s", dsn3, err) } + defer db3.Close() + + cleanupSql := "DROP TABLE IF EXISTS test" - dbt := &DBTest{t, db} - dbt2 := &DBTest{t, db2} - dbt3 := &DBTest{t, db3} for _, test := range tests { - test(dbt) - dbt.db.Exec("DROP TABLE IF EXISTS test") + test := test + t.Run("default", func(t *testing.T) { + dbt := &DBTest{t, db} + t.Cleanup(func() { + db.Exec(cleanupSql) + }) + test(dbt) + }) if db2 != nil { - test(dbt2) - dbt2.db.Exec("DROP TABLE IF EXISTS test") + t.Run("interpolateParams", func(t *testing.T) { + dbt2 := &DBTest{t, db2} + t.Cleanup(func() { + db2.Exec(cleanupSql) + }) + test(dbt2) + }) } - if db3 != nil { + t.Run("compress", func(t *testing.T) { + dbt3 := &DBTest{t, db3} + t.Cleanup(func() { + db3.Exec(cleanupSql) + }) test(dbt3) - dbt3.db.Exec("DROP TABLE IF EXISTS test") + }) + } +} + +// runTestsParallel runs the tests in parallel with a separate database connection for each test. +func runTestsParallel(t *testing.T, dsn string, tests ...func(dbt *DBTest, tableName string)) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + newTableName := func(t *testing.T) string { + t.Helper() + var buf [8]byte + if _, err := rand.Read(buf[:]); err != nil { + t.Fatal(err) + } + return fmt.Sprintf("test_%x", buf[:]) + } + + t.Parallel() + for _, test := range tests { + test := test + + t.Run("default", func(t *testing.T) { + t.Parallel() + + tableName := newTableName(t) + db, err := sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + t.Cleanup(func() { + db.Exec("DROP TABLE IF EXISTS " + tableName) + db.Close() + }) + + dbt := &DBTest{t, db} + test(dbt, tableName) + }) + + dsn2 := dsn + "&interpolateParams=true" + if _, err := ParseDSN(dsn2); err == errInvalidDSNUnsafeCollation { + t.Run("interpolateParams", func(t *testing.T) { + t.Parallel() + + tableName := newTableName(t) + db, err := sql.Open("mysql", dsn2) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + t.Cleanup(func() { + db.Exec("DROP TABLE IF EXISTS " + tableName) + db.Close() + }) + + dbt := &DBTest{t, db} + test(dbt, tableName) + }) } } } func (dbt *DBTest) fail(method, query string, err error) { + dbt.Helper() if len(query) > 300 { query = "[query too large to print]" } dbt.Fatalf("error on %s %s: %s", method, query, err.Error()) } -func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) { +func (dbt *DBTest) mustExec(query string, args ...any) (res sql.Result) { + dbt.Helper() res, err := dbt.db.Exec(query, args...) if err != nil { dbt.fail("exec", query, err) @@ -191,7 +277,8 @@ func (dbt *DBTest) mustExec(query string, args ...interface{}) (res sql.Result) return res } -func (dbt *DBTest) mustQuery(query string, args ...interface{}) (rows *sql.Rows) { +func (dbt *DBTest) mustQuery(query string, args ...any) (rows *sql.Rows) { + dbt.Helper() rows, err := dbt.db.Query(query, args...) if err != nil { dbt.fail("query", query, err) @@ -211,7 +298,7 @@ func maybeSkip(t *testing.T, err error, skipErrno uint16) { } func TestEmptyQuery(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { // just a comment, no query rows := dbt.mustQuery("--") defer rows.Close() @@ -223,20 +310,20 @@ func TestEmptyQuery(t *testing.T) { } func TestCRUD(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { // Create Table - dbt.mustExec("CREATE TABLE test (value BOOL)") + dbt.mustExec("CREATE TABLE " + tbl + " (value BOOL)") // Test for unexpected data var out bool - rows := dbt.mustQuery("SELECT * FROM test") + rows := dbt.mustQuery("SELECT * FROM " + tbl) if rows.Next() { dbt.Error("unexpected data in empty table") } rows.Close() // Create Data - res := dbt.mustExec("INSERT INTO test VALUES (1)") + res := dbt.mustExec("INSERT INTO " + tbl + " VALUES (1)") count, err := res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -254,7 +341,7 @@ func TestCRUD(t *testing.T) { } // Read - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if true != out { @@ -270,7 +357,7 @@ func TestCRUD(t *testing.T) { rows.Close() // Update - res = dbt.mustExec("UPDATE test SET value = ? WHERE value = ?", false, true) + res = dbt.mustExec("UPDATE "+tbl+" SET value = ? WHERE value = ?", false, true) count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -280,7 +367,7 @@ func TestCRUD(t *testing.T) { } // Check Update - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if false != out { @@ -296,7 +383,7 @@ func TestCRUD(t *testing.T) { rows.Close() // Delete - res = dbt.mustExec("DELETE FROM test WHERE value = ?", false) + res = dbt.mustExec("DELETE FROM "+tbl+" WHERE value = ?", false) count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -306,7 +393,7 @@ func TestCRUD(t *testing.T) { } // Check for unexpected rows - res = dbt.mustExec("DELETE FROM test") + res = dbt.mustExec("DELETE FROM " + tbl) count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -317,6 +404,51 @@ func TestCRUD(t *testing.T) { }) } +// TestNumbers test that selecting numeric columns. +// Both of textRows and binaryRows should return same type and value. +func TestNumbersToAny(t *testing.T) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (id INT PRIMARY KEY, b BOOL, i8 TINYINT, " + + "i16 SMALLINT, i32 INT, i64 BIGINT, f32 FLOAT, f64 DOUBLE, iu32 INT UNSIGNED)") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1, true, 127, 32767, 2147483647, 9223372036854775807, 1.25, 2.5, 4294967295)") + + // Use binaryRows for interpolateParams=false and textRows for interpolateParams=true. + rows := dbt.mustQuery("SELECT b, i8, i16, i32, i64, f32, f64, iu32 FROM "+tbl+" WHERE id=?", 1) + if !rows.Next() { + dbt.Fatal("no data") + } + var b, i8, i16, i32, i64, f32, f64, iu32 any + err := rows.Scan(&b, &i8, &i16, &i32, &i64, &f32, &f64, &iu32) + if err != nil { + dbt.Fatal(err) + } + if b.(int64) != 1 { + dbt.Errorf("b != 1") + } + if i8.(int64) != 127 { + dbt.Errorf("i8 != 127") + } + if i16.(int64) != 32767 { + dbt.Errorf("i16 != 32767") + } + if i32.(int64) != 2147483647 { + dbt.Errorf("i32 != 2147483647") + } + if i64.(int64) != 9223372036854775807 { + dbt.Errorf("i64 != 9223372036854775807") + } + if f32.(float32) != 1.25 { + dbt.Errorf("f32 != 1.25") + } + if f64.(float64) != 2.5 { + dbt.Errorf("f64 != 2.5") + } + if iu32.(int64) != 4294967295 { + dbt.Errorf("iu32 != 4294967295") + } + }) +} + func TestMultiQuery(t *testing.T) { runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { // Create Table @@ -347,8 +479,8 @@ func TestMultiQuery(t *testing.T) { rows := dbt.mustQuery("SELECT value FROM test WHERE id=1;") if rows.Next() { rows.Scan(&out) - if 5 != out { - dbt.Errorf("5 != %d", out) + if out != 5 { + dbt.Errorf("expected 5, got %d", out) } if rows.Next() { @@ -363,7 +495,7 @@ func TestMultiQuery(t *testing.T) { } func TestInt(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [5]string{"TINYINT", "SMALLINT", "MEDIUMINT", "INT", "BIGINT"} in := int64(42) var out int64 @@ -371,11 +503,11 @@ func TestInt(t *testing.T) { // SIGNED for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ")") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (?)", in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -386,16 +518,16 @@ func TestInt(t *testing.T) { } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } // UNSIGNED ZEROFILL for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + " ZEROFILL)") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + " ZEROFILL)") - dbt.mustExec("INSERT INTO test VALUES (?)", in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -406,21 +538,21 @@ func TestInt(t *testing.T) { } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestFloat32(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [2]string{"FLOAT", "DOUBLE"} in := float32(42.23) var out float32 var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ")") + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -430,21 +562,21 @@ func TestFloat32(t *testing.T) { dbt.Errorf("%s: no data", v) } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestFloat64(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [2]string{"FLOAT", "DOUBLE"} var expected float64 = 42.23 var out float64 var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (42.23)") - rows = dbt.mustQuery("SELECT value FROM test") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ")") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (42.23)") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if expected != out { @@ -454,21 +586,21 @@ func TestFloat64(t *testing.T) { dbt.Errorf("%s: no data", v) } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestFloat64Placeholder(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [2]string{"FLOAT", "DOUBLE"} var expected float64 = 42.23 var out float64 var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (id int, value " + v + ")") - dbt.mustExec("INSERT INTO test VALUES (1, 42.23)") - rows = dbt.mustQuery("SELECT value FROM test WHERE id = ?", 1) + dbt.mustExec("CREATE TABLE " + tbl + " (id int, value " + v + ")") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1, 42.23)") + rows = dbt.mustQuery("SELECT value FROM "+tbl+" WHERE id = ?", 1) if rows.Next() { rows.Scan(&out) if expected != out { @@ -478,24 +610,24 @@ func TestFloat64Placeholder(t *testing.T) { dbt.Errorf("%s: no data", v) } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } }) } func TestString(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { types := [6]string{"CHAR(255)", "VARCHAR(255)", "TINYTEXT", "TEXT", "MEDIUMTEXT", "LONGTEXT"} in := "κόσμε üöäßñóùéàâÿœ'îë Árvíztűrő いろはにほへとちりぬるを イロハニホヘト דג סקרן чащах น่าฟังเอย" var out string var rows *sql.Rows for _, v := range types { - dbt.mustExec("CREATE TABLE test (value " + v + ") CHARACTER SET utf8") + dbt.mustExec("CREATE TABLE " + tbl + " (value " + v + ") CHARACTER SET utf8") - dbt.mustExec("INSERT INTO test VALUES (?)", in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in != out { @@ -506,11 +638,11 @@ func TestString(t *testing.T) { } rows.Close() - dbt.mustExec("DROP TABLE IF EXISTS test") + dbt.mustExec("DROP TABLE IF EXISTS " + tbl) } // BLOB - dbt.mustExec("CREATE TABLE test (id int, value BLOB) CHARACTER SET utf8") + dbt.mustExec("CREATE TABLE " + tbl + " (id int, value BLOB) CHARACTER SET utf8") id := 2 in = "Lorem ipsum dolor sit amet, consetetur sadipscing elitr, " + @@ -521,9 +653,9 @@ func TestString(t *testing.T) { "sed diam nonumy eirmod tempor invidunt ut labore et dolore magna aliquyam erat, " + "sed diam voluptua. At vero eos et accusam et justo duo dolores et ea rebum. " + "Stet clita kasd gubergren, no sea takimata sanctus est Lorem ipsum dolor sit amet." - dbt.mustExec("INSERT INTO test VALUES (?, ?)", id, in) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?, ?)", id, in) - err := dbt.db.QueryRow("SELECT value FROM test WHERE id = ?", id).Scan(&out) + err := dbt.db.QueryRow("SELECT value FROM "+tbl+" WHERE id = ?", id).Scan(&out) if err != nil { dbt.Fatalf("Error on BLOB-Query: %s", err.Error()) } else if out != in { @@ -533,7 +665,7 @@ func TestString(t *testing.T) { } func TestRawBytes(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { v1 := []byte("aaa") v2 := []byte("bbb") rows := dbt.mustQuery("SELECT ?, ?", v1, v2) @@ -562,7 +694,7 @@ func TestRawBytes(t *testing.T) { } func TestRawMessage(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { v1 := json.RawMessage("{}") v2 := json.RawMessage("[]") rows := dbt.mustQuery("SELECT ?, ?", v1, v2) @@ -593,14 +725,14 @@ func (tv testValuer) Value() (driver.Value, error) { } func TestValuer(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { in := testValuer{"a_value"} var out string var rows *sql.Rows - dbt.mustExec("CREATE TABLE test (value VARCHAR(255)) CHARACTER SET utf8") - dbt.mustExec("INSERT INTO test VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM test") + dbt.mustExec("CREATE TABLE " + tbl + " (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) + rows = dbt.mustQuery("SELECT value FROM " + tbl) if rows.Next() { rows.Scan(&out) if in.value != out { @@ -610,8 +742,6 @@ func TestValuer(t *testing.T) { dbt.Errorf("Valuer: no data") } rows.Close() - - dbt.mustExec("DROP TABLE IF EXISTS test") }) } @@ -628,15 +758,15 @@ func (tv testValuerWithValidation) Value() (driver.Value, error) { } func TestValuerWithValidation(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { in := testValuerWithValidation{"a_value"} var out string var rows *sql.Rows - dbt.mustExec("CREATE TABLE testValuer (value VARCHAR(255)) CHARACTER SET utf8") - dbt.mustExec("INSERT INTO testValuer VALUES (?)", in) + dbt.mustExec("CREATE TABLE " + tbl + " (value VARCHAR(255)) CHARACTER SET utf8") + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?)", in) - rows = dbt.mustQuery("SELECT value FROM testValuer") + rows = dbt.mustQuery("SELECT value FROM " + tbl) defer rows.Close() if rows.Next() { @@ -648,19 +778,17 @@ func TestValuerWithValidation(t *testing.T) { dbt.Errorf("Valuer: no data") } - if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", testValuerWithValidation{""}); err == nil { + if _, err := dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", testValuerWithValidation{""}); err == nil { dbt.Errorf("Failed to check valuer error") } - if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", nil); err != nil { + if _, err := dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", nil); err != nil { dbt.Errorf("Failed to check nil") } - if _, err := dbt.db.Exec("INSERT INTO testValuer VALUES (?)", map[string]bool{}); err == nil { + if _, err := dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", map[string]bool{}); err == nil { dbt.Errorf("Failed to check not valuer") } - - dbt.mustExec("DROP TABLE IF EXISTS testValuer") }) } @@ -737,7 +865,7 @@ func (t timeTest) run(dbt *DBTest, dbtype, tlayout string, mode timeMode) { dbt.Errorf("%s [%s]: %s", dbtype, mode, err) return } - var dst interface{} + var dst any err = rows.Scan(&dst) if err != nil { dbt.Errorf("%s [%s]: %s", dbtype, mode, err) @@ -768,7 +896,7 @@ func (t timeTest) run(dbt *DBTest, dbtype, tlayout string, mode timeMode) { t.s, val.Format(tlayout), ) default: - fmt.Printf("%#v\n", []interface{}{dbtype, tlayout, mode, t.s, t.t}) + fmt.Printf("%#v\n", []any{dbtype, tlayout, mode, t.s, t.t}) dbt.Errorf("%s [%s]: unhandled type %T (is '%v')", dbtype, mode, val, val, @@ -850,12 +978,16 @@ func TestDateTime(t *testing.T) { var err error rows, err = dbt.db.Query(`SELECT cast("00:00:00.1" as TIME(1)) = "00:00:00.1"`) if err == nil { - rows.Scan(µsecsSupported) + if rows.Next() { + rows.Scan(µsecsSupported) + } rows.Close() } rows, err = dbt.db.Query(`SELECT cast("0000-00-00" as DATE) = "0000-00-00"`) if err == nil { - rows.Scan(&zeroDateSupported) + if rows.Next() { + rows.Scan(&zeroDateSupported) + } rows.Close() } for _, setups := range testcases { @@ -894,7 +1026,7 @@ func TestTimestampMicros(t *testing.T) { f0 := format[:19] f1 := format[:21] f6 := format[:26] - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { // check if microseconds are supported. // Do not use timestamp(x) for that check - before 5.5.6, x would mean display width // and not precision. @@ -909,7 +1041,7 @@ func TestTimestampMicros(t *testing.T) { return } _, err := dbt.db.Exec(` - CREATE TABLE test ( + CREATE TABLE ` + tbl + ` ( value0 TIMESTAMP NOT NULL DEFAULT '` + f0 + `', value1 TIMESTAMP(1) NOT NULL DEFAULT '` + f1 + `', value6 TIMESTAMP(6) NOT NULL DEFAULT '` + f6 + `' @@ -918,10 +1050,10 @@ func TestTimestampMicros(t *testing.T) { if err != nil { dbt.Error(err) } - defer dbt.mustExec("DROP TABLE IF EXISTS test") - dbt.mustExec("INSERT INTO test SET value0=?, value1=?, value6=?", f0, f1, f6) + defer dbt.mustExec("DROP TABLE IF EXISTS " + tbl) + dbt.mustExec("INSERT INTO "+tbl+" SET value0=?, value1=?, value6=?", f0, f1, f6) var res0, res1, res6 string - rows := dbt.mustQuery("SELECT * FROM test") + rows := dbt.mustQuery("SELECT * FROM " + tbl) defer rows.Close() if !rows.Next() { dbt.Errorf("test contained no selectable values") @@ -943,7 +1075,7 @@ func TestTimestampMicros(t *testing.T) { } func TestNULL(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { nullStmt, err := dbt.db.Prepare("SELECT NULL") if err != nil { dbt.Fatal(err) @@ -1075,12 +1207,12 @@ func TestNULL(t *testing.T) { } // Insert NULL - dbt.mustExec("CREATE TABLE test (dummmy1 int, value int, dummy2 int)") + dbt.mustExec("CREATE TABLE " + tbl + " (dummmy1 int, value int, dummy2 int)") - dbt.mustExec("INSERT INTO test VALUES (?, ?, ?)", 1, nil, 2) + dbt.mustExec("INSERT INTO "+tbl+" VALUES (?, ?, ?)", 1, nil, 2) - var out interface{} - rows := dbt.mustQuery("SELECT * FROM test") + var out any + rows := dbt.mustQuery("SELECT * FROM " + tbl) defer rows.Close() if rows.Next() { rows.Scan(&out) @@ -1104,7 +1236,7 @@ func TestUint64(t *testing.T) { shigh = int64(uhigh) stop = ^shigh ) - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { stmt, err := dbt.db.Prepare(`SELECT ?, ?, ? ,?, ?, ?, ?, ?`) if err != nil { dbt.Fatal(err) @@ -1157,8 +1289,7 @@ func TestLongData(t *testing.T) { var rows *sql.Rows // Long text data - const nonDataQueryLen = 28 // length query w/o value - inS := in[:maxAllowedPacketSize-nonDataQueryLen] + inS := in[:maxAllowedPacketSize-100] dbt.mustExec("INSERT INTO test VALUES('" + inS + "')") rows = dbt.mustQuery("SELECT value FROM test") defer rows.Close() @@ -1168,7 +1299,7 @@ func TestLongData(t *testing.T) { dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(inS), len(out)) } if rows.Next() { - dbt.Error("LONGBLOB: unexpexted row") + dbt.Error("LONGBLOB: unexpected row") } } else { dbt.Fatalf("LONGBLOB: no data") @@ -1187,7 +1318,7 @@ func TestLongData(t *testing.T) { dbt.Fatalf("LONGBLOB: length in: %d, length out: %d", len(in), len(out)) } if rows.Next() { - dbt.Error("LONGBLOB: unexpexted row") + dbt.Error("LONGBLOB: unexpected row") } } else { if err = rows.Err(); err != nil { @@ -1245,7 +1376,7 @@ func TestLoadData(t *testing.T) { dbt.mustExec("CREATE TABLE test (id INT NOT NULL PRIMARY KEY, value TEXT NOT NULL) CHARACTER SET utf8") // Local File - file, err := ioutil.TempFile("", "gotest") + file, err := os.CreateTemp("", "gotest") defer os.Remove(file.Name()) if err != nil { dbt.Fatal(err) @@ -1263,7 +1394,7 @@ func TestLoadData(t *testing.T) { dbt.Fatalf("unexpected row count: got %d, want 0", count) } - // Then fille File with data and try to load it + // Then fill File with data and try to load it file.WriteString("1\ta string\n2\ta string containing a \\t\n3\ta string containing a \\n\n4\ta string containing both \\t\\n\n") file.Close() dbt.mustExec(fmt.Sprintf("LOAD DATA LOCAL INFILE %q INTO TABLE test", file.Name())) @@ -1294,18 +1425,18 @@ func TestLoadData(t *testing.T) { _, err = dbt.db.Exec("LOAD DATA LOCAL INFILE 'Reader::doesnotexist' INTO TABLE test") if err == nil { dbt.Fatal("load non-existent Reader didn't fail") - } else if err.Error() != "Reader 'doesnotexist' is not registered" { + } else if err.Error() != "reader 'doesnotexist' is not registered" { dbt.Fatal(err.Error()) } }) } -func TestFoundRows(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") - dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") +func TestFoundRows1(t *testing.T) { + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO " + tbl + " (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") - res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") + res := dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 0") count, err := res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1313,7 +1444,7 @@ func TestFoundRows(t *testing.T) { if count != 2 { dbt.Fatalf("Expected 2 affected rows, got %d", count) } - res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + res = dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 1") count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1322,11 +1453,14 @@ func TestFoundRows(t *testing.T) { dbt.Fatalf("Expected 2 affected rows, got %d", count) } }) - runTests(t, dsn+"&clientFoundRows=true", func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (id INT NOT NULL ,data INT NOT NULL)") - dbt.mustExec("INSERT INTO test (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") +} - res := dbt.mustExec("UPDATE test SET data = 1 WHERE id = 0") +func TestFoundRows2(t *testing.T) { + runTestsParallel(t, dsn+"&clientFoundRows=true", func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (id INT NOT NULL ,data INT NOT NULL)") + dbt.mustExec("INSERT INTO " + tbl + " (id, data) VALUES (0, 0),(0, 0),(1, 0),(1, 0),(1, 1)") + + res := dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 0") count, err := res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1334,7 +1468,7 @@ func TestFoundRows(t *testing.T) { if count != 2 { dbt.Fatalf("Expected 2 matched rows, got %d", count) } - res = dbt.mustExec("UPDATE test SET data = 1 WHERE id = 1") + res = dbt.mustExec("UPDATE " + tbl + " SET data = 1 WHERE id = 1") count, err = res.RowsAffected() if err != nil { dbt.Fatalf("res.RowsAffected() returned error: %s", err.Error()) @@ -1402,6 +1536,7 @@ func TestReuseClosedConnection(t *testing.T) { if err != nil { t.Fatalf("error preparing statement: %s", err.Error()) } + //lint:ignore SA1019 this is a test _, err = stmt.Exec(nil) if err != nil { t.Fatalf("error executing statement: %s", err.Error()) @@ -1416,6 +1551,7 @@ func TestReuseClosedConnection(t *testing.T) { t.Errorf("panic after reusing a closed connection: %v", err) } }() + //lint:ignore SA1019 this is a test _, err = stmt.Exec(nil) if err != nil && err != driver.ErrBadConn { t.Errorf("unexpected error '%s', expected '%s'", @@ -1458,7 +1594,7 @@ func TestCharset(t *testing.T) { } func TestFailingCharset(t *testing.T) { - runTests(t, dsn+"&charset=none", func(dbt *DBTest) { + runTestsParallel(t, dsn+"&charset=none", func(dbt *DBTest, _ string) { // run query to really establish connection... _, err := dbt.db.Exec("SELECT 1") if err == nil { @@ -1507,7 +1643,7 @@ func TestCollation(t *testing.T) { } func TestColumnsWithAlias(t *testing.T) { - runTests(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest) { + runTestsParallel(t, dsn+"&columnsWithAlias=true", func(dbt *DBTest, _ string) { rows := dbt.mustQuery("SELECT 1 AS A") defer rows.Close() cols, _ := rows.Columns() @@ -1531,7 +1667,7 @@ func TestColumnsWithAlias(t *testing.T) { } func TestRawBytesResultExceedsBuffer(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { // defaultBufSize from buffer.go expected := strings.Repeat("abc", defaultBufSize) @@ -1590,7 +1726,7 @@ func TestTimezoneConversion(t *testing.T) { // Special cases func TestRowsClose(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { rows, err := dbt.db.Query("SELECT 1") if err != nil { dbt.Fatal(err) @@ -1615,7 +1751,7 @@ func TestRowsClose(t *testing.T) { // dangling statements // http://code.google.com/p/go/issues/detail?id=3865 func TestCloseStmtBeforeRows(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { stmt, err := dbt.db.Prepare("SELECT 1") if err != nil { dbt.Fatal(err) @@ -1656,7 +1792,7 @@ func TestCloseStmtBeforeRows(t *testing.T) { // It is valid to have multiple Rows for the same Stmt // http://code.google.com/p/go/issues/detail?id=3734 func TestStmtMultiRows(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { stmt, err := dbt.db.Prepare("SELECT 1 UNION SELECT 0") if err != nil { dbt.Fatal(err) @@ -1782,7 +1918,7 @@ func TestPreparedManyCols(t *testing.T) { // create more parameters than fit into the buffer // which will take nil-values - params := make([]interface{}, numParams) + params := make([]any, numParams) rows, err := stmt.Query(params...) if err != nil { dbt.Fatal(err) @@ -1807,13 +1943,13 @@ func TestConcurrent(t *testing.T) { } runTests(t, dsn, func(dbt *DBTest) { - var version string - if err := dbt.db.QueryRow("SELECT @@version").Scan(&version); err != nil { - dbt.Fatalf("%s", err.Error()) - } - if strings.Contains(strings.ToLower(version), "mariadb") { - t.Skip(`TODO: "fix commands out of sync. Did you run multiple statements at once?" on MariaDB`) - } + // var version string + // if err := dbt.db.QueryRow("SELECT @@version").Scan(&version); err != nil { + // dbt.Fatal(err) + // } + // if strings.Contains(strings.ToLower(version), "mariadb") { + // t.Skip(`TODO: "fix commands out of sync. Did you run multiple statements at once?" on MariaDB`) + // } var max int err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) @@ -1829,7 +1965,7 @@ func TestConcurrent(t *testing.T) { var fatalError string var once sync.Once - fatalf := func(s string, vals ...interface{}) { + fatalf := func(s string, vals ...any) { once.Do(func() { fatalError = fmt.Sprintf(s, vals...) }) @@ -1840,7 +1976,6 @@ func TestConcurrent(t *testing.T) { defer wg.Done() tx, err := dbt.db.Begin() - atomic.AddInt32(&remaining, -1) if err != nil { if err.Error() != "Error 1040: Too many connections" { @@ -1850,7 +1985,7 @@ func TestConcurrent(t *testing.T) { } // keep the connection busy until all connections are open - for remaining > 0 { + for atomic.AddInt32(&remaining, -1) > 0 { if _, err = tx.Exec("DO 1"); err != nil { fatalf("error on conn %d: %s", id, err.Error()) return @@ -1867,7 +2002,7 @@ func TestConcurrent(t *testing.T) { }(i) } - // wait until all conections are open + // wait until all connections are open wg.Wait() if fatalError != "" { @@ -1883,7 +2018,7 @@ func testDialError(t *testing.T, dialErr error, expectErr error) { return nil, dialErr }) - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + db, err := sql.Open(driverNameTest, fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -1916,13 +2051,13 @@ func TestCustomDial(t *testing.T) { t.Skipf("MySQL server not running on %s", netAddr) } - // our custom dial function which justs wraps net.Dial here + // our custom dial function which just wraps net.Dial here RegisterDialContext("mydial", func(ctx context.Context, addr string) (net.Conn, error) { var d net.Dialer return d.DialContext(ctx, prot, addr) }) - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) + db, err := sql.Open(driverNameTest, fmt.Sprintf("%s:%s@mydial(%s)/%s?timeout=30s", user, pass, addr, dbname)) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -1933,6 +2068,40 @@ func TestCustomDial(t *testing.T) { } } +func TestBeforeConnect(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + // dbname is set in the BeforeConnect handle + cfg, err := ParseDSN(fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, pass, netAddr, "_")) + if err != nil { + t.Fatalf("error parsing DSN: %v", err) + } + + cfg.Apply(BeforeConnect(func(ctx context.Context, c *Config) error { + c.DBName = dbname + return nil + })) + + connector, err := NewConnector(cfg) + if err != nil { + t.Fatalf("error creating connector: %v", err) + } + + db := sql.OpenDB(connector) + defer db.Close() + + var connectedDb string + err = db.QueryRow("SELECT DATABASE();").Scan(&connectedDb) + if err != nil { + t.Fatalf("error executing query: %v", err) + } + if connectedDb != dbname { + t.Fatalf("expected to connect to DB %s, but connected to %s instead", dbname, connectedDb) + } +} + func TestSQLInjection(t *testing.T) { createTest := func(arg string) func(dbt *DBTest) { return func(dbt *DBTest) { @@ -1995,7 +2164,7 @@ func TestInsertRetrieveEscapedData(t *testing.T) { func TestUnixSocketAuthFail(t *testing.T) { runTests(t, dsn, func(dbt *DBTest) { // Save the current logger so we can restore it. - oldLogger := errLog + oldLogger := defaultLogger // Set a new logger so we can capture its output. buffer := bytes.NewBuffer(make([]byte, 0, 64)) @@ -2020,7 +2189,7 @@ func TestUnixSocketAuthFail(t *testing.T) { } t.Logf("socket: %s", socket) badDSN := fmt.Sprintf("%s:%s@unix(%s)/%s?timeout=30s", user, badPass, socket, dbname) - db, err := sql.Open("mysql", badDSN) + db, err := sql.Open(driverNameTest, badDSN) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -2155,11 +2324,51 @@ func TestRejectReadOnly(t *testing.T) { } func TestPing(t *testing.T) { + ctx := context.Background() runTests(t, dsn, func(dbt *DBTest) { if err := dbt.db.Ping(); err != nil { dbt.fail("Ping", "Ping", err) } }) + + runTests(t, dsn, func(dbt *DBTest) { + conn, err := dbt.db.Conn(ctx) + if err != nil { + dbt.fail("db", "Conn", err) + } + + // Check that affectedRows and insertIds are cleared after each call. + conn.Raw(func(conn any) error { + c := conn.(*mysqlConn) + + // Issue a query that sets affectedRows and insertIds. + q, err := c.Query(`SELECT 1`, nil) + if err != nil { + dbt.fail("Conn", "Query", err) + } + if got, want := c.result.affectedRows, []int64{0}; !reflect.DeepEqual(got, want) { + dbt.Fatalf("bad affectedRows: got %v, want=%v", got, want) + } + if got, want := c.result.insertIds, []int64{0}; !reflect.DeepEqual(got, want) { + dbt.Fatalf("bad insertIds: got %v, want=%v", got, want) + } + q.Close() + + // Verify that Ping() clears both fields. + for i := 0; i < 2; i++ { + if err := c.Ping(ctx); err != nil { + dbt.fail("Pinger", "Ping", err) + } + if got, want := c.result.affectedRows, []int64(nil); !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + if got, want := c.result.insertIds, []int64(nil); !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + } + return nil + }) + }) } // See Issue #799 @@ -2169,7 +2378,7 @@ func TestEmptyPassword(t *testing.T) { } dsn := fmt.Sprintf("%s:%s@%s/%s?timeout=30s", user, "", netAddr, dbname) - db, err := sql.Open("mysql", dsn) + db, err := sql.Open(driverNameTest, dsn) if err == nil { defer db.Close() err = db.Ping() @@ -2379,10 +2588,47 @@ func TestMultiResultSetNoSelect(t *testing.T) { }) } +func TestExecMultipleResults(t *testing.T) { + ctx := context.Background() + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + CREATE TABLE test ( + id INT NOT NULL AUTO_INCREMENT, + value VARCHAR(255), + PRIMARY KEY (id) + )`) + conn, err := dbt.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + conn.Raw(func(conn any) error { + //lint:ignore SA1019 this is a test + ex := conn.(driver.Execer) + res, err := ex.Exec(` + INSERT INTO test (value) VALUES ('a'), ('b'); + INSERT INTO test (value) VALUES ('c'), ('d'), ('e'); + `, nil) + if err != nil { + t.Fatalf("insert statements failed: %v", err) + } + mres := res.(Result) + if got, want := mres.AllRowsAffected(), []int64{2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad AllRowsAffected: got %v, want=%v", got, want) + } + // For INSERTs containing multiple rows, LAST_INSERT_ID() returns the + // first inserted ID, not the last. + if got, want := mres.AllLastInsertIds(), []int64{1, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad AllLastInsertIds: got %v, want %v", got, want) + } + return nil + }) + }) +} + // tests if rows are set in a proper state if some results were ignored before // calling rows.NextResultSet. func TestSkipResults(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { rows := dbt.mustQuery("SELECT 1, 2") defer rows.Close() @@ -2400,8 +2646,44 @@ func TestSkipResults(t *testing.T) { }) } +func TestQueryMultipleResults(t *testing.T) { + ctx := context.Background() + runTestsWithMultiStatement(t, dsn, func(dbt *DBTest) { + dbt.mustExec(` + CREATE TABLE test ( + id INT NOT NULL AUTO_INCREMENT, + value VARCHAR(255), + PRIMARY KEY (id) + )`) + conn, err := dbt.db.Conn(ctx) + if err != nil { + t.Fatalf("failed to connect: %v", err) + } + conn.Raw(func(conn any) error { + //lint:ignore SA1019 this is a test + qr := conn.(driver.Queryer) + c := conn.(*mysqlConn) + + // Demonstrate that repeated queries reset the affectedRows + for i := 0; i < 2; i++ { + _, err := qr.Query(` + INSERT INTO test (value) VALUES ('a'), ('b'); + INSERT INTO test (value) VALUES ('c'), ('d'), ('e'); + `, nil) + if err != nil { + t.Fatalf("insert statements failed: %v", err) + } + if got, want := c.result.affectedRows, []int64{2, 3}; !reflect.DeepEqual(got, want) { + t.Errorf("bad affectedRows: got %v, want=%v", got, want) + } + } + return nil + }) + }) +} + func TestPingContext(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { ctx, cancel := context.WithCancel(context.Background()) cancel() if err := dbt.db.PingContext(ctx); err != context.Canceled { @@ -2411,8 +2693,8 @@ func TestPingContext(t *testing.T) { } func TestContextCancelExec(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) // Delay execution for just a bit until db.ExecContext has begun. @@ -2420,7 +2702,7 @@ func TestContextCancelExec(t *testing.T) { // This query will be canceled. startTime := time.Now() - if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } if d := time.Since(startTime); d > 500*time.Millisecond { @@ -2432,7 +2714,7 @@ func TestContextCancelExec(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2440,14 +2722,14 @@ func TestContextCancelExec(t *testing.T) { } // Context is already canceled, so error should come before execution. - if _, err := dbt.db.ExecContext(ctx, "INSERT INTO test VALUES (1)"); err == nil { + if _, err := dbt.db.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (1)"); err == nil { dbt.Error("expected error") } else if err.Error() != "context canceled" { dbt.Fatalf("unexpected error: %s", err) } // The second insert query will fail, so the table has no changes. - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { @@ -2457,8 +2739,8 @@ func TestContextCancelExec(t *testing.T) { } func TestContextCancelQuery(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) // Delay execution for just a bit until db.ExecContext has begun. @@ -2466,7 +2748,7 @@ func TestContextCancelQuery(t *testing.T) { // This query will be canceled. startTime := time.Now() - if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } if d := time.Since(startTime); d > 500*time.Millisecond { @@ -2478,7 +2760,7 @@ func TestContextCancelQuery(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2486,12 +2768,12 @@ func TestContextCancelQuery(t *testing.T) { } // Context is already canceled, so error should come before execution. - if _, err := dbt.db.QueryContext(ctx, "INSERT INTO test VALUES (1)"); err != context.Canceled { + if _, err := dbt.db.QueryContext(ctx, "INSERT INTO "+tbl+" VALUES (1)"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } // The second insert query will fail, so the table has no changes. - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { @@ -2501,12 +2783,12 @@ func TestContextCancelQuery(t *testing.T) { } func TestContextCancelQueryRow(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") - dbt.mustExec("INSERT INTO test VALUES (1), (2), (3)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") + dbt.mustExec("INSERT INTO " + tbl + " VALUES (1), (2), (3)") ctx, cancel := context.WithCancel(context.Background()) - rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM test") + rows, err := dbt.db.QueryContext(ctx, "SELECT v FROM "+tbl) if err != nil { dbt.Fatalf("%s", err.Error()) } @@ -2534,7 +2816,7 @@ func TestContextCancelQueryRow(t *testing.T) { } func TestContextCancelPrepare(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { + runTestsParallel(t, dsn, func(dbt *DBTest, _ string) { ctx, cancel := context.WithCancel(context.Background()) cancel() if _, err := dbt.db.PrepareContext(ctx, "SELECT 1"); err != context.Canceled { @@ -2544,10 +2826,10 @@ func TestContextCancelPrepare(t *testing.T) { } func TestContextCancelStmtExec(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) - stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))") if err != nil { dbt.Fatalf("unexpected error: %v", err) } @@ -2569,7 +2851,7 @@ func TestContextCancelStmtExec(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2579,10 +2861,10 @@ func TestContextCancelStmtExec(t *testing.T) { } func TestContextCancelStmtQuery(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) - stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO test VALUES (SLEEP(1))") + stmt, err := dbt.db.PrepareContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))") if err != nil { dbt.Fatalf("unexpected error: %v", err) } @@ -2604,7 +2886,7 @@ func TestContextCancelStmtQuery(t *testing.T) { // Check how many times the query is executed. var v int - if err := dbt.db.QueryRow("SELECT COUNT(*) FROM test").Scan(&v); err != nil { + if err := dbt.db.QueryRow("SELECT COUNT(*) FROM " + tbl).Scan(&v); err != nil { dbt.Fatalf("%s", err.Error()) } if v != 1 { // TODO: need to kill the query, and v should be 0. @@ -2618,8 +2900,8 @@ func TestContextCancelBegin(t *testing.T) { t.Skip(`FIXME: it sometime fails with "expected driver.ErrBadConn, got sql: connection is already closed" on windows and macOS`) } - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) conn, err := dbt.db.Conn(ctx) if err != nil { @@ -2636,7 +2918,7 @@ func TestContextCancelBegin(t *testing.T) { // This query will be canceled. startTime := time.Now() - if _, err := tx.ExecContext(ctx, "INSERT INTO test VALUES (SLEEP(1))"); err != context.Canceled { + if _, err := tx.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (SLEEP(1))"); err != context.Canceled { dbt.Errorf("expected context.Canceled, got %v", err) } if d := time.Since(startTime); d > 500*time.Millisecond { @@ -2674,8 +2956,8 @@ func TestContextCancelBegin(t *testing.T) { } func TestContextBeginIsolationLevel(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2693,17 +2975,17 @@ func TestContextBeginIsolationLevel(t *testing.T) { dbt.Fatal(err) } - _, err = tx1.ExecContext(ctx, "INSERT INTO test VALUES (1)") + _, err = tx1.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (1)") if err != nil { dbt.Fatal(err) } var v int - row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + row := tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tbl) if err := row.Scan(&v); err != nil { dbt.Fatal(err) } - // Because writer transaction wasn't commited yet, it should be available + // Because writer transaction wasn't committed yet, it should be available if v != 0 { dbt.Errorf("expected val to be 0, got %d", v) } @@ -2713,11 +2995,11 @@ func TestContextBeginIsolationLevel(t *testing.T) { dbt.Fatal(err) } - row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + row = tx2.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tbl) if err := row.Scan(&v); err != nil { dbt.Fatal(err) } - // Data written by writer transaction is already commited, it should be selectable + // Data written by writer transaction is already committed, it should be selectable if v != 1 { dbt.Errorf("expected val to be 1, got %d", v) } @@ -2726,8 +3008,8 @@ func TestContextBeginIsolationLevel(t *testing.T) { } func TestContextBeginReadOnly(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (v INTEGER)") + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (v INTEGER)") ctx, cancel := context.WithCancel(context.Background()) defer cancel() @@ -2742,14 +3024,14 @@ func TestContextBeginReadOnly(t *testing.T) { } // INSERT queries fail in a READ ONLY transaction. - _, err = tx.ExecContext(ctx, "INSERT INTO test VALUES (1)") + _, err = tx.ExecContext(ctx, "INSERT INTO "+tbl+" VALUES (1)") if _, ok := err.(*MySQLError); !ok { dbt.Errorf("expected MySQLError, got %v", err) } // SELECT queries can be executed. var v int - row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM test") + row := tx.QueryRowContext(ctx, "SELECT COUNT(*) FROM "+tbl) if err := row.Scan(&v); err != nil { dbt.Fatal(err) } @@ -2778,13 +3060,18 @@ func TestRowsColumnTypes(t *testing.T) { nd1 := sql.NullTime{Time: time.Date(2006, 01, 02, 0, 0, 0, 0, time.UTC), Valid: true} nd2 := sql.NullTime{Time: time.Date(2006, 03, 04, 0, 0, 0, 0, time.UTC), Valid: true} ndNULL := sql.NullTime{Time: time.Time{}, Valid: false} - rbNULL := sql.RawBytes(nil) - rb0 := sql.RawBytes("0") - rb42 := sql.RawBytes("42") - rbTest := sql.RawBytes("Test") - rb0pad4 := sql.RawBytes("0\x00\x00\x00") // BINARY right-pads values with 0x00 - rbx0 := sql.RawBytes("\x00") - rbx42 := sql.RawBytes("\x42") + bNULL := []byte(nil) + nsNULL := sql.NullString{String: "", Valid: false} + // Helper function to build NullString from string literal. + ns := func(s string) sql.NullString { return sql.NullString{String: s, Valid: true} } + ns0 := ns("0") + b0 := []byte("0") + b42 := []byte("42") + nsTest := ns("Test") + bTest := []byte("Test") + b0pad4 := []byte("0\x00\x00\x00") // BINARY right-pads values with 0x00 + bx0 := []byte("\x00") + bx42 := []byte("\x42") var columns = []struct { name string @@ -2795,51 +3082,54 @@ func TestRowsColumnTypes(t *testing.T) { precision int64 // 0 if not ok scale int64 valuesIn [3]string - valuesOut [3]interface{} + valuesOut [3]any }{ - {"bit8null", "BIT(8)", "BIT", scanTypeRawBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]interface{}{rbx0, rbNULL, rbx42}}, - {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]interface{}{niNULL, ni1, ni0}}, - {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]interface{}{int8(1), int8(0), int8(0)}}, - {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, - {"smallint", "SMALLINT NOT NULL", "SMALLINT", scanTypeInt16, false, 0, 0, [3]string{"0", "-32768", "32767"}, [3]interface{}{int16(0), int16(-32768), int16(32767)}}, - {"smallintnull", "SMALLINT", "SMALLINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, - {"int3null", "INT(3)", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]interface{}{ni0, niNULL, ni42}}, - {"int7", "INT(7) NOT NULL", "INT", scanTypeInt32, false, 0, 0, [3]string{"0", "-1337", "42"}, [3]interface{}{int32(0), int32(-1337), int32(42)}}, - {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]interface{}{ni0, ni42, niNULL}}, - {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]interface{}{int64(0), int64(65535), int64(-42)}}, - {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]interface{}{niNULL, ni1, ni42}}, - {"tinyuint", "TINYINT UNSIGNED NOT NULL", "UNSIGNED TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]interface{}{uint8(0), uint8(255), uint8(42)}}, - {"smalluint", "SMALLINT UNSIGNED NOT NULL", "UNSIGNED SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint16(0), uint16(65535), uint16(42)}}, - {"biguint", "BIGINT UNSIGNED NOT NULL", "UNSIGNED BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]interface{}{uint64(0), uint64(65535), uint64(42)}}, - {"uint13", "INT(13) UNSIGNED NOT NULL", "UNSIGNED INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]interface{}{uint32(0), uint32(1337), uint32(42)}}, - {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float32(0), float32(42), float32(13.37)}}, - {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, - {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, - {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]interface{}{float64(0), float64(42), float64(13.37)}}, - {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]interface{}{nf0, nfNULL, nf1337}}, - {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), sql.RawBytes("13.370000"), sql.RawBytes("1234.123456")}}, - {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeRawBytes, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.000000"), rbNULL, sql.RawBytes("1234.123456")}}, - {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), sql.RawBytes("13.3700"), sql.RawBytes("1234.1235")}}, - {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeRawBytes, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]interface{}{sql.RawBytes("0.0000"), rbNULL, sql.RawBytes("1234.1235")}}, - {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeRawBytes, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]interface{}{rb0, sql.RawBytes("13"), sql.RawBytes("-12345")}}, - {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeRawBytes, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]interface{}{rb0, rbNULL, sql.RawBytes("-12345")}}, - {"char25null", "CHAR(25)", "CHAR", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"binary4null", "BINARY(4)", "BINARY", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0pad4, rbNULL, rbTest}}, - {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"tinytextnull", "TINYTEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"blobnull", "BLOB", "BLOB", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"textnull", "TEXT", "TEXT", scanTypeRawBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]interface{}{rb0, rbNULL, rbTest}}, - {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeRawBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]interface{}{rb0, rbTest, rb42}}, - {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt0, nt0}}, - {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt2}}, - {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]interface{}{nt0, nt1, nt6}}, - {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]interface{}{nd1, ndNULL, nd2}}, - {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]interface{}{uint16(2006), uint16(2000), uint16(1994)}}, + {"bit8null", "BIT(8)", "BIT", scanTypeBytes, true, 0, 0, [3]string{"0x0", "NULL", "0x42"}, [3]any{bx0, bNULL, bx42}}, + {"boolnull", "BOOL", "TINYINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "true", "0"}, [3]any{niNULL, ni1, ni0}}, + {"bool", "BOOL NOT NULL", "TINYINT", scanTypeInt8, false, 0, 0, [3]string{"1", "0", "FALSE"}, [3]any{int8(1), int8(0), int8(0)}}, + {"intnull", "INTEGER", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]any{ni0, niNULL, ni42}}, + {"smallint", "SMALLINT NOT NULL", "SMALLINT", scanTypeInt16, false, 0, 0, [3]string{"0", "-32768", "32767"}, [3]any{int16(0), int16(-32768), int16(32767)}}, + {"smallintnull", "SMALLINT", "SMALLINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]any{ni0, niNULL, ni42}}, + {"int3null", "INT(3)", "INT", scanTypeNullInt, true, 0, 0, [3]string{"0", "NULL", "42"}, [3]any{ni0, niNULL, ni42}}, + {"int7", "INT(7) NOT NULL", "INT", scanTypeInt32, false, 0, 0, [3]string{"0", "-1337", "42"}, [3]any{int32(0), int32(-1337), int32(42)}}, + {"mediumintnull", "MEDIUMINT", "MEDIUMINT", scanTypeNullInt, true, 0, 0, [3]string{"0", "42", "NULL"}, [3]any{ni0, ni42, niNULL}}, + {"bigint", "BIGINT NOT NULL", "BIGINT", scanTypeInt64, false, 0, 0, [3]string{"0", "65535", "-42"}, [3]any{int64(0), int64(65535), int64(-42)}}, + {"bigintnull", "BIGINT", "BIGINT", scanTypeNullInt, true, 0, 0, [3]string{"NULL", "1", "42"}, [3]any{niNULL, ni1, ni42}}, + {"tinyuint", "TINYINT UNSIGNED NOT NULL", "UNSIGNED TINYINT", scanTypeUint8, false, 0, 0, [3]string{"0", "255", "42"}, [3]any{uint8(0), uint8(255), uint8(42)}}, + {"smalluint", "SMALLINT UNSIGNED NOT NULL", "UNSIGNED SMALLINT", scanTypeUint16, false, 0, 0, [3]string{"0", "65535", "42"}, [3]any{uint16(0), uint16(65535), uint16(42)}}, + {"biguint", "BIGINT UNSIGNED NOT NULL", "UNSIGNED BIGINT", scanTypeUint64, false, 0, 0, [3]string{"0", "65535", "42"}, [3]any{uint64(0), uint64(65535), uint64(42)}}, + {"mediumuint", "MEDIUMINT UNSIGNED NOT NULL", "UNSIGNED MEDIUMINT", scanTypeUint32, false, 0, 0, [3]string{"0", "16777215", "42"}, [3]any{uint32(0), uint32(16777215), uint32(42)}}, + {"uint13", "INT(13) UNSIGNED NOT NULL", "UNSIGNED INT", scanTypeUint32, false, 0, 0, [3]string{"0", "1337", "42"}, [3]any{uint32(0), uint32(1337), uint32(42)}}, + {"float", "FLOAT NOT NULL", "FLOAT", scanTypeFloat32, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]any{float32(0), float32(42), float32(13.37)}}, + {"floatnull", "FLOAT", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]any{nf0, nfNULL, nf1337}}, + {"float74null", "FLOAT(7,4)", "FLOAT", scanTypeNullFloat, true, math.MaxInt64, 4, [3]string{"0", "NULL", "13.37"}, [3]any{nf0, nfNULL, nf1337}}, + {"double", "DOUBLE NOT NULL", "DOUBLE", scanTypeFloat64, false, math.MaxInt64, math.MaxInt64, [3]string{"0", "42", "13.37"}, [3]any{float64(0), float64(42), float64(13.37)}}, + {"doublenull", "DOUBLE", "DOUBLE", scanTypeNullFloat, true, math.MaxInt64, math.MaxInt64, [3]string{"0", "NULL", "13.37"}, [3]any{nf0, nfNULL, nf1337}}, + {"decimal1", "DECIMAL(10,6) NOT NULL", "DECIMAL", scanTypeString, false, 10, 6, [3]string{"0", "13.37", "1234.123456"}, [3]any{"0.000000", "13.370000", "1234.123456"}}, + {"decimal1null", "DECIMAL(10,6)", "DECIMAL", scanTypeNullString, true, 10, 6, [3]string{"0", "NULL", "1234.123456"}, [3]any{ns("0.000000"), nsNULL, ns("1234.123456")}}, + {"decimal2", "DECIMAL(8,4) NOT NULL", "DECIMAL", scanTypeString, false, 8, 4, [3]string{"0", "13.37", "1234.123456"}, [3]any{"0.0000", "13.3700", "1234.1235"}}, + {"decimal2null", "DECIMAL(8,4)", "DECIMAL", scanTypeNullString, true, 8, 4, [3]string{"0", "NULL", "1234.123456"}, [3]any{ns("0.0000"), nsNULL, ns("1234.1235")}}, + {"decimal3", "DECIMAL(5,0) NOT NULL", "DECIMAL", scanTypeString, false, 5, 0, [3]string{"0", "13.37", "-12345.123456"}, [3]any{"0", "13", "-12345"}}, + {"decimal3null", "DECIMAL(5,0)", "DECIMAL", scanTypeNullString, true, 5, 0, [3]string{"0", "NULL", "-12345.123456"}, [3]any{ns0, nsNULL, ns("-12345")}}, + {"char25null", "CHAR(25)", "CHAR", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]any{ns0, nsNULL, nsTest}}, + {"varchar42", "VARCHAR(42) NOT NULL", "VARCHAR", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]any{"0", "Test", "42"}}, + {"binary4null", "BINARY(4)", "BINARY", scanTypeBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]any{b0pad4, bNULL, bTest}}, + {"varbinary42", "VARBINARY(42) NOT NULL", "VARBINARY", scanTypeBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]any{b0, bTest, b42}}, + {"tinyblobnull", "TINYBLOB", "BLOB", scanTypeBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]any{b0, bNULL, bTest}}, + {"tinytextnull", "TINYTEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]any{ns0, nsNULL, nsTest}}, + {"blobnull", "BLOB", "BLOB", scanTypeBytes, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]any{b0, bNULL, bTest}}, + {"textnull", "TEXT", "TEXT", scanTypeNullString, true, 0, 0, [3]string{"0", "NULL", "'Test'"}, [3]any{ns0, nsNULL, nsTest}}, + {"mediumblob", "MEDIUMBLOB NOT NULL", "BLOB", scanTypeBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]any{b0, bTest, b42}}, + {"mediumtext", "MEDIUMTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]any{"0", "Test", "42"}}, + {"longblob", "LONGBLOB NOT NULL", "BLOB", scanTypeBytes, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]any{b0, bTest, b42}}, + {"longtext", "LONGTEXT NOT NULL", "TEXT", scanTypeString, false, 0, 0, [3]string{"0", "'Test'", "42"}, [3]any{"0", "Test", "42"}}, + {"datetime", "DATETIME", "DATETIME", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]any{nt0, nt0, nt0}}, + {"datetime2", "DATETIME(2)", "DATETIME", scanTypeNullTime, true, 2, 2, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]any{nt0, nt1, nt2}}, + {"datetime6", "DATETIME(6)", "DATETIME", scanTypeNullTime, true, 6, 6, [3]string{"'2006-01-02 15:04:05'", "'2006-01-02 15:04:05.1'", "'2006-01-02 15:04:05.111111'"}, [3]any{nt0, nt1, nt6}}, + {"date", "DATE", "DATE", scanTypeNullTime, true, 0, 0, [3]string{"'2006-01-02'", "NULL", "'2006-03-04'"}, [3]any{nd1, ndNULL, nd2}}, + {"year", "YEAR NOT NULL", "YEAR", scanTypeUint16, false, 0, 0, [3]string{"2006", "2000", "1994"}, [3]any{uint16(2006), uint16(2000), uint16(1994)}}, + {"enum", "ENUM('', 'v1', 'v2')", "ENUM", scanTypeNullString, true, 0, 0, [3]string{"''", "'v1'", "'v2'"}, [3]any{ns(""), ns("v1"), ns("v2")}}, + {"set", "set('', 'v1', 'v2')", "SET", scanTypeNullString, true, 0, 0, [3]string{"''", "'v1'", "'v1,v2'"}, [3]any{ns(""), ns("v1"), ns("v1,v2")}}, } schema := "" @@ -2945,8 +3235,11 @@ func TestRowsColumnTypes(t *testing.T) { continue } } - - values := make([]interface{}, len(tt)) + // Avoid panic caused by nil scantype. + if t.Failed() { + return + } + values := make([]any, len(tt)) for i := range values { values[i] = reflect.New(types[i]).Interface() } @@ -2956,14 +3249,10 @@ func TestRowsColumnTypes(t *testing.T) { if err != nil { t.Fatalf("failed to scan values in %v", err) } - for j := range values { - value := reflect.ValueOf(values[j]).Elem().Interface() + for j, value := range values { + value := reflect.ValueOf(value).Elem().Interface() if !reflect.DeepEqual(value, columns[j].valuesOut[i]) { - if columns[j].scanType == scanTypeRawBytes { - t.Errorf("row %d, column %d: %v != %v", i, j, string(value.(sql.RawBytes)), string(columns[j].valuesOut[i].(sql.RawBytes))) - } else { - t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) - } + t.Errorf("row %d, column %d: %v != %v", i, j, value, columns[j].valuesOut[i]) } } i++ @@ -2979,9 +3268,9 @@ func TestRowsColumnTypes(t *testing.T) { } func TestValuerWithValueReceiverGivenNilValue(t *testing.T) { - runTests(t, dsn, func(dbt *DBTest) { - dbt.mustExec("CREATE TABLE test (value VARCHAR(255))") - dbt.db.Exec("INSERT INTO test VALUES (?)", (*testValuer)(nil)) + runTestsParallel(t, dsn, func(dbt *DBTest, tbl string) { + dbt.mustExec("CREATE TABLE " + tbl + " (value VARCHAR(255))") + dbt.db.Exec("INSERT INTO "+tbl+" VALUES (?)", (*testValuer)(nil)) // This test will panic on the INSERT if ConvertValue() does not check for typed nil before calling Value() }) } @@ -3015,27 +3304,28 @@ func TestRawBytesAreNotModified(t *testing.T) { rows, err := dbt.db.QueryContext(ctx, `SELECT id, value FROM test`) if err != nil { - t.Fatal(err) + dbt.Fatal(err) } + defer rows.Close() var b int var raw sql.RawBytes - for rows.Next() { - if err := rows.Scan(&b, &raw); err != nil { - t.Fatal(err) - } + if !rows.Next() { + dbt.Fatal("expected at least one row") + } + if err := rows.Scan(&b, &raw); err != nil { + dbt.Fatal(err) + } - before := string(raw) - // Ensure cancelling the query does not corrupt the contents of `raw` - cancel() - time.Sleep(time.Microsecond * 100) - after := string(raw) + before := string(raw) + // Ensure cancelling the query does not corrupt the contents of `raw` + cancel() + time.Sleep(time.Microsecond * 100) + after := string(raw) - if before != after { - t.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) - } + if before != after { + dbt.Fatalf("the backing storage for sql.RawBytes has been modified (i=%v)", i) } - rows.Close() }() } }) @@ -3058,7 +3348,7 @@ func TestConnectorObeysDialTimeouts(t *testing.T) { return d.DialContext(ctx, prot, addr) }) - db, err := sql.Open("mysql", fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) + db, err := sql.Open(driverNameTest, fmt.Sprintf("%s:%s@dialctxtest(%s)/%s?timeout=30s", user, pass, addr, dbname)) if err != nil { t.Fatalf("error connecting: %s", err.Error()) } @@ -3209,3 +3499,149 @@ func TestConnectorTimeoutsWatchCancel(t *testing.T) { t.Errorf("connection not closed") } } + +func TestConnectionAttributes(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + + defaultAttrs := []string{ + connAttrClientName, + connAttrOS, + connAttrPlatform, + connAttrPid, + connAttrServerHost, + } + host, _, _ := net.SplitHostPort(addr) + defaultAttrValues := []string{ + connAttrClientNameValue, + connAttrOSValue, + connAttrPlatformValue, + strconv.Itoa(os.Getpid()), + host, + } + + customAttrs := []string{"attr1", "fo/o"} + customAttrValues := []string{"value1", "bo/o"} + + customAttrStrs := make([]string, len(customAttrs)) + for i := range customAttrs { + customAttrStrs[i] = fmt.Sprintf("%s:%s", customAttrs[i], customAttrValues[i]) + } + dsn += "&connectionAttributes=" + url.QueryEscape(strings.Join(customAttrStrs, ",")) + + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open(driverNameTest, dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + + queryString := "SELECT ATTR_NAME, ATTR_VALUE FROM performance_schema.session_account_connect_attrs WHERE PROCESSLIST_ID = CONNECTION_ID()" + rows := dbt.mustQuery(queryString) + defer rows.Close() + + rowsMap := make(map[string]string) + for rows.Next() { + var attrName, attrValue string + rows.Scan(&attrName, &attrValue) + rowsMap[attrName] = attrValue + } + + connAttrs := append(append([]string{}, defaultAttrs...), customAttrs...) + expectedAttrValues := append(append([]string{}, defaultAttrValues...), customAttrValues...) + for i := range connAttrs { + if gotValue := rowsMap[connAttrs[i]]; gotValue != expectedAttrValues[i] { + dbt.Errorf("expected %q, got %q", expectedAttrValues[i], gotValue) + } + } +} + +func TestErrorInMultiResult(t *testing.T) { + if !available { + t.Skipf("MySQL server not running on %s", netAddr) + } + // https://github.com/go-sql-driver/mysql/issues/1361 + var db *sql.DB + if _, err := ParseDSN(dsn); err != errInvalidDSNUnsafeCollation { + db, err = sql.Open("mysql", dsn) + if err != nil { + t.Fatalf("error connecting: %s", err.Error()) + } + defer db.Close() + } + + dbt := &DBTest{t, db} + query := ` +CREATE PROCEDURE test_proc1() +BEGIN + SELECT 1,2; + SELECT 3,4; + SIGNAL SQLSTATE '10000' SET MESSAGE_TEXT = "some error", MYSQL_ERRNO = 10000; +END; +` + runCallCommand(dbt, query, "test_proc1") +} + +func runCallCommand(dbt *DBTest, query, name string) { + dbt.mustExec(fmt.Sprintf("DROP PROCEDURE IF EXISTS %s", name)) + dbt.mustExec(query) + defer dbt.mustExec("DROP PROCEDURE " + name) + rows, err := dbt.db.Query(fmt.Sprintf("CALL %s", name)) + if err != nil { + return + } + defer rows.Close() + + for rows.Next() { + } + for rows.NextResultSet() { + for rows.Next() { + } + } +} + +func TestIssue1567(t *testing.T) { + // enable TLS. + runTests(t, dsn+"&tls=skip-verify", func(dbt *DBTest) { + var max int + err := dbt.db.QueryRow("SELECT @@max_connections").Scan(&max) + if err != nil { + dbt.Fatalf("%s", err.Error()) + } + + // disable connection pooling. + // data race happens when new connection is created. + dbt.db.SetMaxIdleConns(0) + + // estimate round trip time. + start := time.Now() + if err := dbt.db.PingContext(context.Background()); err != nil { + t.Fatal(err) + } + rtt := time.Since(start) + if rtt <= 0 { + // In some environments, rtt may become 0, so set it to at least 1ms. + rtt = time.Millisecond + } + + count := 1000 + if testing.Short() { + count = 10 + } + if count > max { + count = max + } + + for i := 0; i < count; i++ { + timeout := time.Duration(mrand.Int63n(int64(rtt))) + ctx, cancel := context.WithTimeout(context.Background(), timeout) + dbt.db.PingContext(ctx) + cancel() + } + }) +} diff --git a/dsn.go b/dsn.go index a306d66a3..ecf62567a 100644 --- a/dsn.go +++ b/dsn.go @@ -10,6 +10,7 @@ package mysql import ( "bytes" + "context" "crypto/rsa" "crypto/tls" "errors" @@ -34,51 +35,125 @@ var ( // If a new Config is created instead of being parsed from a DSN string, // the NewConfig function should be used, which sets default values. type Config struct { - User string // Username - Passwd string // Password (requires User) - Net string // Network type - Addr string // Network address (requires Net) - DBName string // Database name - Params map[string]string // Connection parameters - Collation string // Connection collation - Loc *time.Location // Location for time.Time values - MaxAllowedPacket int // Max packet size allowed - ServerPubKey string // Server public key name - pubKey *rsa.PublicKey // Server public key - TLSConfig string // TLS configuration name - tls *tls.Config // TLS configuration - Timeout time.Duration // Dial timeout - ReadTimeout time.Duration // I/O read timeout - WriteTimeout time.Duration // I/O write timeout - - AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE - AllowCleartextPasswords bool // Allows the cleartext client side plugin - AllowNativePasswords bool // Allows the native password authentication method - AllowOldPasswords bool // Allows the old insecure password method - CheckConnLiveness bool // Check connections for liveness before using them - ClientFoundRows bool // Return number of matching rows instead of rows changed - ColumnsWithAlias bool // Prepend table alias to column names - InterpolateParams bool // Interpolate placeholders into query string - MultiStatements bool // Allow multiple statements in one query - ParseTime bool // Parse time values to time.Time - RejectReadOnly bool // Reject read-only connections + // non boolean fields + + User string // Username + Passwd string // Password (requires User) + Net string // Network (e.g. "tcp", "tcp6", "unix". default: "tcp") + Addr string // Address (default: "127.0.0.1:3306" for "tcp" and "/tmp/mysql.sock" for "unix") + DBName string // Database name + Params map[string]string // Connection parameters + ConnectionAttributes string // Connection Attributes, comma-delimited string of user-defined "key:value" pairs + Collation string // Connection collation. When set, this will be set in SET NAMES COLLATE query + Loc *time.Location // Location for time.Time values + MaxAllowedPacket int // Max packet size allowed + ServerPubKey string // Server public key name + TLSConfig string // TLS configuration name + TLS *tls.Config // TLS configuration, its priority is higher than TLSConfig + Timeout time.Duration // Dial timeout + ReadTimeout time.Duration // I/O read timeout + WriteTimeout time.Duration // I/O write timeout + Logger Logger // Logger + // DialFunc specifies the dial function for creating connections + DialFunc func(ctx context.Context, network, addr string) (net.Conn, error) + + // boolean fields + + AllowAllFiles bool // Allow all files to be used with LOAD DATA LOCAL INFILE + AllowCleartextPasswords bool // Allows the cleartext client side plugin + AllowFallbackToPlaintext bool // Allows fallback to unencrypted connection if server does not support TLS + AllowNativePasswords bool // Allows the native password authentication method + AllowOldPasswords bool // Allows the old insecure password method + CheckConnLiveness bool // Check connections for liveness before using them + ClientFoundRows bool // Return number of matching rows instead of rows changed + ColumnsWithAlias bool // Prepend table alias to column names + InterpolateParams bool // Interpolate placeholders into query string + MultiStatements bool // Allow multiple statements in one query + ParseTime bool // Parse time values to time.Time + RejectReadOnly bool // Reject read-only connections + + // unexported fields. new options should be come here. + // boolean first. alphabetical order. + + compress bool // Enable zlib compression + + beforeConnect func(context.Context, *Config) error // Invoked before a connection is established + pubKey *rsa.PublicKey // Server public key + timeTruncate time.Duration // Truncate time.Time values to the specified duration + charsets []string // Connection charset. When set, this will be set in SET NAMES query } +// Functional Options Pattern +// https://dave.cheney.net/2014/10/17/functional-options-for-friendly-apis +type Option func(*Config) error + // NewConfig creates a new Config and sets default values. func NewConfig() *Config { - return &Config{ - Collation: defaultCollation, + cfg := &Config{ Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, + Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, } + return cfg +} + +// Apply applies the given options to the Config object. +func (c *Config) Apply(opts ...Option) error { + for _, opt := range opts { + err := opt(c) + if err != nil { + return err + } + } + return nil +} + +// TimeTruncate sets the time duration to truncate time.Time values in +// query parameters. +func TimeTruncate(d time.Duration) Option { + return func(cfg *Config) error { + cfg.timeTruncate = d + return nil + } +} + +// BeforeConnect sets the function to be invoked before a connection is established. +func BeforeConnect(fn func(context.Context, *Config) error) Option { + return func(cfg *Config) error { + cfg.beforeConnect = fn + return nil + } +} + +// EnableCompress sets the compression mode. +func EnableCompression(yes bool) Option { + return func(cfg *Config) error { + cfg.compress = yes + return nil + } +} + +// Charset sets the connection charset and collation. +// +// charset is the connection charset. +// collation is the connection collation. It can be null or empty string. +// +// When collation is not specified, `SET NAMES ` command is sent when the connection is established. +// When collation is specified, `SET NAMES COLLATE ` command is sent when the connection is established. +func Charset(charset, collation string) Option { + return func(cfg *Config) error { + cfg.charsets = []string{charset} + cfg.Collation = collation + return nil + } } func (cfg *Config) Clone() *Config { cp := *cfg - if cp.tls != nil { - cp.tls = cfg.tls.Clone() + if cp.TLS != nil { + cp.TLS = cfg.TLS.Clone() } if len(cp.Params) > 0 { cp.Params = make(map[string]string, len(cfg.Params)) @@ -96,7 +171,7 @@ func (cfg *Config) Clone() *Config { } func (cfg *Config) normalize() error { - if cfg.InterpolateParams && unsafeCollations[cfg.Collation] { + if cfg.InterpolateParams && cfg.Collation != "" && unsafeCollations[cfg.Collation] { return errInvalidDSNUnsafeCollation } @@ -119,24 +194,29 @@ func (cfg *Config) normalize() error { cfg.Addr = ensureHavePort(cfg.Addr) } - switch cfg.TLSConfig { - case "false", "": - // don't set anything - case "true": - cfg.tls = &tls.Config{} - case "skip-verify", "preferred": - cfg.tls = &tls.Config{InsecureSkipVerify: true} - default: - cfg.tls = getTLSConfigClone(cfg.TLSConfig) - if cfg.tls == nil { - return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + if cfg.TLS == nil { + switch cfg.TLSConfig { + case "false", "": + // don't set anything + case "true": + cfg.TLS = &tls.Config{} + case "skip-verify": + cfg.TLS = &tls.Config{InsecureSkipVerify: true} + case "preferred": + cfg.TLS = &tls.Config{InsecureSkipVerify: true} + cfg.AllowFallbackToPlaintext = true + default: + cfg.TLS = getTLSConfigClone(cfg.TLSConfig) + if cfg.TLS == nil { + return errors.New("invalid value / unknown config name: " + cfg.TLSConfig) + } } } - if cfg.tls != nil && cfg.tls.ServerName == "" && !cfg.tls.InsecureSkipVerify { + if cfg.TLS != nil && cfg.TLS.ServerName == "" && !cfg.TLS.InsecureSkipVerify { host, _, err := net.SplitHostPort(cfg.Addr) if err == nil { - cfg.tls.ServerName = host + cfg.TLS.ServerName = host } } @@ -147,6 +227,10 @@ func (cfg *Config) normalize() error { } } + if cfg.Logger == nil { + cfg.Logger = defaultLogger + } + return nil } @@ -165,6 +249,8 @@ func writeDSNParam(buf *bytes.Buffer, hasParam *bool, name, value string) { // FormatDSN formats the given Config into a DSN string which can be passed to // the driver. +// +// Note: use [NewConnector] and [database/sql.OpenDB] to open a connection from a [*Config]. func (cfg *Config) FormatDSN() string { var buf bytes.Buffer @@ -190,7 +276,7 @@ func (cfg *Config) FormatDSN() string { // /dbname buf.WriteByte('/') - buf.WriteString(cfg.DBName) + buf.WriteString(url.PathEscape(cfg.DBName)) // [?param1=value1&...¶mN=valueN] hasParam := false @@ -204,6 +290,10 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "allowCleartextPasswords", "true") } + if cfg.AllowFallbackToPlaintext { + writeDSNParam(&buf, &hasParam, "allowFallbackToPlaintext", "true") + } + if !cfg.AllowNativePasswords { writeDSNParam(&buf, &hasParam, "allowNativePasswords", "false") } @@ -220,7 +310,11 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "clientFoundRows", "true") } - if col := cfg.Collation; col != defaultCollation && len(col) > 0 { + if charsets := cfg.charsets; len(charsets) > 0 { + writeDSNParam(&buf, &hasParam, "charset", strings.Join(charsets, ",")) + } + + if col := cfg.Collation; col != "" { writeDSNParam(&buf, &hasParam, "collation", col) } @@ -228,6 +322,14 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "columnsWithAlias", "true") } + if cfg.ConnectionAttributes != "" { + writeDSNParam(&buf, &hasParam, "connectionAttributes", url.QueryEscape(cfg.ConnectionAttributes)) + } + + if cfg.compress { + writeDSNParam(&buf, &hasParam, "compress", "true") + } + if cfg.InterpolateParams { writeDSNParam(&buf, &hasParam, "interpolateParams", "true") } @@ -244,6 +346,10 @@ func (cfg *Config) FormatDSN() string { writeDSNParam(&buf, &hasParam, "parseTime", "true") } + if cfg.timeTruncate > 0 { + writeDSNParam(&buf, &hasParam, "timeTruncate", cfg.timeTruncate.String()) + } + if cfg.ReadTimeout > 0 { writeDSNParam(&buf, &hasParam, "readTimeout", cfg.ReadTimeout.String()) } @@ -348,7 +454,11 @@ func ParseDSN(dsn string) (cfg *Config, err error) { break } } - cfg.DBName = dsn[i+1 : j] + + dbname := dsn[i+1 : j] + if cfg.DBName, err = url.PathUnescape(dbname); err != nil { + return nil, fmt.Errorf("invalid dbname %q: %w", dbname, err) + } break } @@ -368,13 +478,13 @@ func ParseDSN(dsn string) (cfg *Config, err error) { // Values must be url.QueryEscape'ed func parseDSNParams(cfg *Config, params string) (err error) { for _, v := range strings.Split(params, "&") { - param := strings.SplitN(v, "=", 2) - if len(param) != 2 { + key, value, found := strings.Cut(v, "=") + if !found { continue } // cfg params - switch value := param[1]; param[0] { + switch key { // Disable INFILE allowlist / enable all files case "allowAllFiles": var isBool bool @@ -391,6 +501,14 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // Allow fallback to unencrypted connection if server does not support TLS + case "allowFallbackToPlaintext": + var isBool bool + cfg.AllowFallbackToPlaintext, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } + // Use native password authentication case "allowNativePasswords": var isBool bool @@ -423,6 +541,10 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // charset + case "charset": + cfg.charsets = strings.Split(value, ",") + // Collation case "collation": cfg.Collation = value @@ -436,7 +558,11 @@ func parseDSNParams(cfg *Config, params string) (err error) { // Compression case "compress": - return errors.New("compression not implemented yet") + var isBool bool + cfg.compress, isBool = readBool(value) + if !isBool { + return errors.New("invalid bool value: " + value) + } // Enable client side placeholder substitution case "interpolateParams": @@ -472,6 +598,13 @@ func parseDSNParams(cfg *Config, params string) (err error) { return errors.New("invalid bool value: " + value) } + // time.Time truncation + case "timeTruncate": + cfg.timeTruncate, err = time.ParseDuration(value) + if err != nil { + return fmt.Errorf("invalid timeTruncate value: %v, error: %w", value, err) + } + // I/O read Timeout case "readTimeout": cfg.ReadTimeout, err = time.ParseDuration(value) @@ -536,13 +669,22 @@ func parseDSNParams(cfg *Config, params string) (err error) { if err != nil { return } + + // Connection attributes + case "connectionAttributes": + connectionAttributes, err := url.QueryUnescape(value) + if err != nil { + return fmt.Errorf("invalid connectionAttributes value: %v", err) + } + cfg.ConnectionAttributes = connectionAttributes + default: // lazy init if cfg.Params == nil { cfg.Params = make(map[string]string) } - if cfg.Params[param[0]], err = url.QueryUnescape(value); err != nil { + if cfg.Params[key], err = url.QueryUnescape(value); err != nil { return } } diff --git a/dsn_fuzz_test.go b/dsn_fuzz_test.go new file mode 100644 index 000000000..04c56ad45 --- /dev/null +++ b/dsn_fuzz_test.go @@ -0,0 +1,47 @@ +//go:build go1.18 +// +build go1.18 + +package mysql + +import ( + "net" + "testing" +) + +func FuzzFormatDSN(f *testing.F) { + for _, test := range testDSNs { // See dsn_test.go + f.Add(test.in) + } + + f.Fuzz(func(t *testing.T, dsn1 string) { + // Do not waste resources + if len(dsn1) > 1000 { + t.Skip("ignore: too long") + } + + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Skipf("invalid DSN: %v", err) + } + + dsn2 := cfg1.FormatDSN() + if dsn2 == dsn1 { + return + } + + // Skip known cases of bad config that are not strictly checked by ParseDSN + if _, _, err := net.SplitHostPort(cfg1.Addr); err != nil { + t.Skipf("invalid addr %q: %v", cfg1.Addr, err) + } + + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Fatalf("%q rewritten as %q: %v", dsn1, dsn2, err) + } + + dsn3 := cfg2.FormatDSN() + if dsn3 != dsn2 { + t.Errorf("%q rewritten as %q", dsn2, dsn3) + } + }) +} diff --git a/dsn_test.go b/dsn_test.go index fc6eea9c8..436f77992 100644 --- a/dsn_test.go +++ b/dsn_test.go @@ -22,71 +22,83 @@ var testDSNs = []struct { out *Config }{{ "username:password@protocol(address)/dbname?param=value", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true}, }, { "username:password@protocol(address)/dbname?param=value&columnsWithAlias=true&multiStatements=true", - &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, + &Config{User: "username", Passwd: "password", Net: "protocol", Addr: "address", DBName: "dbname", Params: map[string]string{"param": "value"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ColumnsWithAlias: true, MultiStatements: true}, }, { "user@unix(/path/to/socket)/dbname?charset=utf8", - &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Net: "unix", Addr: "/path/to/socket", DBName: "dbname", charsets: []string{"utf8"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8&tls=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", charsets: []string{"utf8"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "true"}, }, { "user:password@tcp(localhost:5555)/dbname?charset=utf8mb4,utf8&tls=skip-verify", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", Params: map[string]string{"charset": "utf8mb4,utf8"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "localhost:5555", DBName: "dbname", charsets: []string{"utf8mb4", "utf8"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, TLSConfig: "skip-verify"}, }, { "user:password@/dbname?loc=UTC&timeout=30s&readTimeout=1s&writeTimeout=1s&allowAllFiles=1&clientFoundRows=true&allowOldPasswords=TRUE&collation=utf8mb4_unicode_ci&maxAllowedPacket=16777216&tls=false&allowCleartextPasswords=true&parseTime=true&rejectReadOnly=true", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_unicode_ci", Loc: time.UTC, TLSConfig: "false", AllowCleartextPasswords: true, AllowNativePasswords: true, Timeout: 30 * time.Second, ReadTimeout: time.Second, WriteTimeout: time.Second, Logger: defaultLogger, AllowAllFiles: true, AllowOldPasswords: true, CheckConnLiveness: true, ClientFoundRows: true, MaxAllowedPacket: 16777216, ParseTime: true, RejectReadOnly: true}, }, { - "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0", - &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: 0, AllowNativePasswords: false, CheckConnLiveness: false}, + "user:password@/dbname?allowNativePasswords=false&checkConnLiveness=false&maxAllowedPacket=0&allowFallbackToPlaintext=true", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: 0, Logger: defaultLogger, AllowFallbackToPlaintext: true, AllowNativePasswords: false, CheckConnLiveness: false}, }, { "user:p@ss(word)@tcp([de:ad:be:ef::ca:fe]:80)/dbname?loc=Local", - &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Passwd: "p@ss(word)", Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:80", DBName: "dbname", Loc: time.Local, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, +}, { + "/dbname%2Fwithslash", + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname/withslash", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "@/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "/", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "user:p@/ssword@/", - &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{User: "user", Passwd: "p@/ssword", Net: "tcp", Addr: "127.0.0.1:3306", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "unix/?arg=%2Fsome%2Fpath.ext", - &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "unix", Addr: "/tmp/mysql.sock", Params: map[string]string{"arg": "/some/path.ext"}, Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(127.0.0.1)/dbname", - &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, }, { "tcp(de:ad:be:ef::ca:fe)/dbname", - &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Collation: "utf8mb4_general_ci", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, AllowNativePasswords: true, CheckConnLiveness: true}, + &Config{Net: "tcp", Addr: "[de:ad:be:ef::ca:fe]:3306", DBName: "dbname", Loc: time.UTC, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true}, +}, { + "user:password@/dbname?loc=UTC&timeout=30s&parseTime=true&timeTruncate=1h", + &Config{User: "user", Passwd: "password", Net: "tcp", Addr: "127.0.0.1:3306", DBName: "dbname", Loc: time.UTC, Timeout: 30 * time.Second, ParseTime: true, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, timeTruncate: time.Hour}, +}, { + "foo:bar@tcp(192.168.1.50:3307)/baz?timeout=10s&connectionAttributes=program_name:MySQLGoDriver%2FTest,program_version:1.2.3", + &Config{User: "foo", Passwd: "bar", Net: "tcp", Addr: "192.168.1.50:3307", DBName: "baz", Loc: time.UTC, Timeout: 10 * time.Second, MaxAllowedPacket: defaultMaxAllowedPacket, Logger: defaultLogger, AllowNativePasswords: true, CheckConnLiveness: true, ConnectionAttributes: "program_name:MySQLGoDriver/Test,program_version:1.2.3"}, }, } func TestDSNParser(t *testing.T) { for i, tst := range testDSNs { - cfg, err := ParseDSN(tst.in) - if err != nil { - t.Error(err.Error()) - } + t.Run(tst.in, func(t *testing.T) { + cfg, err := ParseDSN(tst.in) + if err != nil { + t.Error(err.Error()) + return + } - // pointer not static - cfg.tls = nil + // pointer not static + cfg.TLS = nil - if !reflect.DeepEqual(cfg, tst.out) { - t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) - } + if !reflect.DeepEqual(cfg, tst.out) { + t.Errorf("%d. ParseDSN(%q) mismatch:\ngot %+v\nwant %+v", i, tst.in, cfg, tst.out) + } + }) } } @@ -100,6 +112,8 @@ func TestDSNParserInvalid(t *testing.T) { "User:pass@tcp(1.2.3.4:3306)", // no trailing slash "net()/", // unknown default addr "user:pass@tcp(127.0.0.1:3306)/db/name", // invalid dbname + "user:password@/dbname?allowFallbackToPlaintext=PREFERRED", // wrong bool flag + "user:password@/dbname?connectionAttributes=attr1:/unescaped/value", // unescaped //"/dbname?arg=/some/unescaped/path", } @@ -112,27 +126,39 @@ func TestDSNParserInvalid(t *testing.T) { func TestDSNReformat(t *testing.T) { for i, tst := range testDSNs { - dsn1 := tst.in - cfg1, err := ParseDSN(dsn1) - if err != nil { - t.Error(err.Error()) - continue - } - cfg1.tls = nil // pointer not static - res1 := fmt.Sprintf("%+v", cfg1) - - dsn2 := cfg1.FormatDSN() - cfg2, err := ParseDSN(dsn2) - if err != nil { - t.Error(err.Error()) - continue - } - cfg2.tls = nil // pointer not static - res2 := fmt.Sprintf("%+v", cfg2) + t.Run(tst.in, func(t *testing.T) { + dsn1 := tst.in + cfg1, err := ParseDSN(dsn1) + if err != nil { + t.Error(err.Error()) + return + } + cfg1.TLS = nil // pointer not static + res1 := fmt.Sprintf("%+v", cfg1) - if res1 != res2 { - t.Errorf("%d. %q does not match %q", i, res2, res1) - } + dsn2 := cfg1.FormatDSN() + if dsn2 != dsn1 { + // Just log + t.Logf("%d. %q reformatted as %q", i, dsn1, dsn2) + } + + cfg2, err := ParseDSN(dsn2) + if err != nil { + t.Error(err.Error()) + return + } + cfg2.TLS = nil // pointer not static + res2 := fmt.Sprintf("%+v", cfg2) + + if res1 != res2 { + t.Errorf("%d. %q does not match %q", i, res2, res1) + } + + dsn3 := cfg2.FormatDSN() + if dsn3 != dsn2 { + t.Errorf("%d. %q does not match %q", i, dsn2, dsn3) + } + }) } } @@ -203,7 +229,7 @@ func TestDSNWithCustomTLS(t *testing.T) { if err != nil { t.Error(err.Error()) - } else if cfg.tls.ServerName != name { + } else if cfg.TLS.ServerName != name { t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, tst) } @@ -214,7 +240,7 @@ func TestDSNWithCustomTLS(t *testing.T) { if err != nil { t.Error(err.Error()) - } else if cfg.tls.ServerName != name { + } else if cfg.TLS.ServerName != name { t.Errorf("did not get the correct ServerName (%s) parsing DSN (%s).", name, tst) } else if tlsCfg.ServerName != "" { t.Errorf("tlsCfg was mutated ServerName (%s) should be empty parsing DSN (%s).", name, tst) @@ -229,11 +255,11 @@ func TestDSNTLSConfig(t *testing.T) { if err != nil { t.Error(err.Error()) } - if cfg.tls == nil { + if cfg.TLS == nil { t.Error("cfg.tls should not be nil") } - if cfg.tls.ServerName != expectedServerName { - t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + if cfg.TLS.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.TLS.ServerName) } dsn = "tcp(example.com)/?tls=true" @@ -241,11 +267,11 @@ func TestDSNTLSConfig(t *testing.T) { if err != nil { t.Error(err.Error()) } - if cfg.tls == nil { + if cfg.TLS == nil { t.Error("cfg.tls should not be nil") } - if cfg.tls.ServerName != expectedServerName { - t.Errorf("cfg.tls.ServerName should be %q, got %q (host without port)", expectedServerName, cfg.tls.ServerName) + if cfg.TLS.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host without port)", expectedServerName, cfg.TLS.ServerName) } } @@ -262,7 +288,7 @@ func TestDSNWithCustomTLSQueryEscape(t *testing.T) { if err != nil { t.Error(err.Error()) - } else if cfg.tls.ServerName != name { + } else if cfg.TLS.ServerName != name { t.Errorf("did not get the correct TLS ServerName (%s) parsing DSN (%s).", name, dsn) } } @@ -335,12 +361,12 @@ func TestCloneConfig(t *testing.T) { t.Errorf("Config.Clone did not create a separate config struct") } - if cfg2.tls.ServerName != expectedServerName { - t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.tls.ServerName) + if cfg2.TLS.ServerName != expectedServerName { + t.Errorf("cfg.tls.ServerName should be %q, got %q (host with port)", expectedServerName, cfg.TLS.ServerName) } - cfg2.tls.ServerName = "example2.com" - if cfg.tls.ServerName == cfg2.tls.ServerName { + cfg2.TLS.ServerName = "example2.com" + if cfg.TLS.ServerName == cfg2.TLS.ServerName { t.Errorf("changed cfg.tls.Server name should not propagate to original Config") } @@ -384,20 +410,20 @@ func TestNormalizeTLSConfig(t *testing.T) { cfg.normalize() - if cfg.tls == nil { + if cfg.TLS == nil { if tc.want != nil { t.Fatal("wanted a tls config but got nil instead") } return } - if cfg.tls.ServerName != tc.want.ServerName { + if cfg.TLS.ServerName != tc.want.ServerName { t.Errorf("tls.ServerName doesn't match (want: '%s', got: '%s')", - tc.want.ServerName, cfg.tls.ServerName) + tc.want.ServerName, cfg.TLS.ServerName) } - if cfg.tls.InsecureSkipVerify != tc.want.InsecureSkipVerify { + if cfg.TLS.InsecureSkipVerify != tc.want.InsecureSkipVerify { t.Errorf("tls.InsecureSkipVerify doesn't match (want: %T, got :%T)", - tc.want.InsecureSkipVerify, cfg.tls.InsecureSkipVerify) + tc.want.InsecureSkipVerify, cfg.TLS.InsecureSkipVerify) } }) } diff --git a/errors.go b/errors.go index 92cc9a361..584617b11 100644 --- a/errors.go +++ b/errors.go @@ -21,46 +21,57 @@ var ( ErrMalformPkt = errors.New("malformed packet") ErrNoTLS = errors.New("TLS requested but server does not support TLS") ErrCleartextPassword = errors.New("this user requires clear text authentication. If you still want to use it, please add 'allowCleartextPasswords=1' to your DSN") - ErrNativePassword = errors.New("this user requires mysql native password authentication.") + ErrNativePassword = errors.New("this user requires mysql native password authentication") ErrOldPassword = errors.New("this user requires old password authentication. If you still want to use it, please add 'allowOldPasswords=1' to your DSN. See also https://github.com/go-sql-driver/mysql/wiki/old_passwords") ErrUnknownPlugin = errors.New("this authentication plugin is not supported") ErrOldProtocol = errors.New("MySQL server does not support required protocol 41+") ErrPktSync = errors.New("commands out of sync. You can't run this command now") ErrPktSyncMul = errors.New("commands out of sync. Did you run multiple statements at once?") - ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the 'max_allowed_packet' variable on the server") + ErrPktTooLarge = errors.New("packet for query is too large. Try adjusting the `Config.MaxAllowedPacket`") ErrBusyBuffer = errors.New("busy buffer") // errBadConnNoWrite is used for connection errors where nothing was sent to the database yet. // If this happens first in a function starting a database interaction, it should be replaced by driver.ErrBadConn - // to trigger a resend. + // to trigger a resend. Use mc.markBadConn(err) to do this. // See https://github.com/go-sql-driver/mysql/pull/302 errBadConnNoWrite = errors.New("bad connection") ) -var errLog = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime|log.Lshortfile)) +var defaultLogger = Logger(log.New(os.Stderr, "[mysql] ", log.Ldate|log.Ltime)) // Logger is used to log critical error messages. type Logger interface { - Print(v ...interface{}) + Print(v ...any) } -// SetLogger is used to set the logger for critical errors. +// NopLogger is a nop implementation of the Logger interface. +type NopLogger struct{} + +// Print implements Logger interface. +func (nl *NopLogger) Print(_ ...any) {} + +// SetLogger is used to set the default logger for critical errors. // The initial logger is os.Stderr. func SetLogger(logger Logger) error { if logger == nil { return errors.New("logger is nil") } - errLog = logger + defaultLogger = logger return nil } // MySQLError is an error type which represents a single MySQL error type MySQLError struct { - Number uint16 - Message string + Number uint16 + SQLState [5]byte + Message string } func (me *MySQLError) Error() string { + if me.SQLState != [5]byte{} { + return fmt.Sprintf("Error %d (%s): %s", me.Number, me.SQLState, me.Message) + } + return fmt.Sprintf("Error %d: %s", me.Number, me.Message) } diff --git a/errors_test.go b/errors_test.go index 3a1aef74d..53d634454 100644 --- a/errors_test.go +++ b/errors_test.go @@ -16,9 +16,9 @@ import ( ) func TestErrorsSetLogger(t *testing.T) { - previous := errLog + previous := defaultLogger defer func() { - errLog = previous + defaultLogger = previous }() // set up logger @@ -28,7 +28,7 @@ func TestErrorsSetLogger(t *testing.T) { // print SetLogger(logger) - errLog.Print("test") + defaultLogger.Print("test") // check result if actual := buffer.String(); actual != expected { @@ -43,13 +43,13 @@ func TestErrorsStrictIgnoreNotes(t *testing.T) { } func TestMySQLErrIs(t *testing.T) { - infraErr := &MySQLError{1234, "the server is on fire"} - otherInfraErr := &MySQLError{1234, "the datacenter is flooded"} + infraErr := &MySQLError{Number: 1234, Message: "the server is on fire"} + otherInfraErr := &MySQLError{Number: 1234, Message: "the datacenter is flooded"} if !errors.Is(infraErr, otherInfraErr) { t.Errorf("expected errors to be the same: %+v %+v", infraErr, otherInfraErr) } - differentCodeErr := &MySQLError{5678, "the server is on fire"} + differentCodeErr := &MySQLError{Number: 5678, Message: "the server is on fire"} if errors.Is(infraErr, differentCodeErr) { t.Fatalf("expected errors to be different: %+v %+v", infraErr, differentCodeErr) } diff --git a/fields.go b/fields.go index e0654a83d..be5cd809a 100644 --- a/fields.go +++ b/fields.go @@ -18,7 +18,7 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeBit: return "BIT" case fieldTypeBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "TEXT" } return "BLOB" @@ -37,6 +37,9 @@ func (mf *mysqlField) typeDatabaseName() string { case fieldTypeGeometry: return "GEOMETRY" case fieldTypeInt24: + if mf.flags&flagUnsigned != 0 { + return "UNSIGNED MEDIUMINT" + } return "MEDIUMINT" case fieldTypeJSON: return "JSON" @@ -46,7 +49,7 @@ func (mf *mysqlField) typeDatabaseName() string { } return "INT" case fieldTypeLongBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "LONGTEXT" } return "LONGBLOB" @@ -56,7 +59,7 @@ func (mf *mysqlField) typeDatabaseName() string { } return "BIGINT" case fieldTypeMediumBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "MEDIUMTEXT" } return "MEDIUMBLOB" @@ -74,7 +77,12 @@ func (mf *mysqlField) typeDatabaseName() string { } return "SMALLINT" case fieldTypeString: - if mf.charSet == collations[binaryCollation] { + if mf.flags&flagEnum != 0 { + return "ENUM" + } else if mf.flags&flagSet != 0 { + return "SET" + } + if mf.charSet == binaryCollationID { return "BINARY" } return "CHAR" @@ -88,43 +96,47 @@ func (mf *mysqlField) typeDatabaseName() string { } return "TINYINT" case fieldTypeTinyBLOB: - if mf.charSet != collations[binaryCollation] { + if mf.charSet != binaryCollationID { return "TINYTEXT" } return "TINYBLOB" case fieldTypeVarChar: - if mf.charSet == collations[binaryCollation] { + if mf.charSet == binaryCollationID { return "VARBINARY" } return "VARCHAR" case fieldTypeVarString: - if mf.charSet == collations[binaryCollation] { + if mf.charSet == binaryCollationID { return "VARBINARY" } return "VARCHAR" case fieldTypeYear: return "YEAR" + case fieldTypeVector: + return "VECTOR" default: return "" } } var ( - scanTypeFloat32 = reflect.TypeOf(float32(0)) - scanTypeFloat64 = reflect.TypeOf(float64(0)) - scanTypeInt8 = reflect.TypeOf(int8(0)) - scanTypeInt16 = reflect.TypeOf(int16(0)) - scanTypeInt32 = reflect.TypeOf(int32(0)) - scanTypeInt64 = reflect.TypeOf(int64(0)) - scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) - scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) - scanTypeNullTime = reflect.TypeOf(sql.NullTime{}) - scanTypeUint8 = reflect.TypeOf(uint8(0)) - scanTypeUint16 = reflect.TypeOf(uint16(0)) - scanTypeUint32 = reflect.TypeOf(uint32(0)) - scanTypeUint64 = reflect.TypeOf(uint64(0)) - scanTypeRawBytes = reflect.TypeOf(sql.RawBytes{}) - scanTypeUnknown = reflect.TypeOf(new(interface{})) + scanTypeFloat32 = reflect.TypeOf(float32(0)) + scanTypeFloat64 = reflect.TypeOf(float64(0)) + scanTypeInt8 = reflect.TypeOf(int8(0)) + scanTypeInt16 = reflect.TypeOf(int16(0)) + scanTypeInt32 = reflect.TypeOf(int32(0)) + scanTypeInt64 = reflect.TypeOf(int64(0)) + scanTypeNullFloat = reflect.TypeOf(sql.NullFloat64{}) + scanTypeNullInt = reflect.TypeOf(sql.NullInt64{}) + scanTypeNullTime = reflect.TypeOf(sql.NullTime{}) + scanTypeUint8 = reflect.TypeOf(uint8(0)) + scanTypeUint16 = reflect.TypeOf(uint16(0)) + scanTypeUint32 = reflect.TypeOf(uint32(0)) + scanTypeUint64 = reflect.TypeOf(uint64(0)) + scanTypeString = reflect.TypeOf("") + scanTypeNullString = reflect.TypeOf(sql.NullString{}) + scanTypeBytes = reflect.TypeOf([]byte{}) + scanTypeUnknown = reflect.TypeOf(new(any)) ) type mysqlField struct { @@ -187,12 +199,18 @@ func (mf *mysqlField) scanType() reflect.Type { } return scanTypeNullFloat + case fieldTypeBit, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, + fieldTypeBLOB, fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeVector: + if mf.charSet == binaryCollationID { + return scanTypeBytes + } + fallthrough case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, - fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, - fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, - fieldTypeTime: - return scanTypeRawBytes + fieldTypeEnum, fieldTypeSet, fieldTypeJSON, fieldTypeTime: + if mf.flags&flagNotNULL != 0 { + return scanTypeString + } + return scanTypeNullString case fieldTypeDate, fieldTypeNewDate, fieldTypeTimestamp, fieldTypeDateTime: diff --git a/fuzz.go b/fuzz.go deleted file mode 100644 index fa75adf6a..000000000 --- a/fuzz.go +++ /dev/null @@ -1,24 +0,0 @@ -// Go MySQL Driver - A MySQL-Driver for Go's database/sql package. -// -// Copyright 2020 The Go-MySQL-Driver Authors. All rights reserved. -// -// This Source Code Form is subject to the terms of the Mozilla Public -// License, v. 2.0. If a copy of the MPL was not distributed with this file, -// You can obtain one at http://mozilla.org/MPL/2.0/. - -// +build gofuzz - -package mysql - -import ( - "database/sql" -) - -func Fuzz(data []byte) int { - db, err := sql.Open("mysql", string(data)) - if err != nil { - return 0 - } - db.Close() - return 1 -} diff --git a/go.mod b/go.mod index 251110478..187aff179 100644 --- a/go.mod +++ b/go.mod @@ -1,3 +1,5 @@ module github.com/go-sql-driver/mysql -go 1.13 +go 1.21.0 + +require filippo.io/edwards25519 v1.1.0 diff --git a/go.sum b/go.sum new file mode 100644 index 000000000..359ca94b4 --- /dev/null +++ b/go.sum @@ -0,0 +1,2 @@ +filippo.io/edwards25519 v1.1.0 h1:FNf4tywRC1HmFuKW5xopWpigGjJKiJSV0Cqo0cJWDaA= +filippo.io/edwards25519 v1.1.0/go.mod h1:BxyFTGdWcka3PhytdK4V28tE5sGfRvvvRV7EaN4VDT4= diff --git a/infile.go b/infile.go index 60effdfc2..453ae091e 100644 --- a/infile.go +++ b/infile.go @@ -17,7 +17,7 @@ import ( ) var ( - fileRegister map[string]bool + fileRegister map[string]struct{} fileRegisterLock sync.RWMutex readerRegister map[string]func() io.Reader readerRegisterLock sync.RWMutex @@ -28,20 +28,19 @@ var ( // Alternatively you can allow the use of all local files with // the DSN parameter 'allowAllFiles=true' // -// filePath := "/home/gopher/data.csv" -// mysql.RegisterLocalFile(filePath) -// err := db.Exec("LOAD DATA LOCAL INFILE '" + filePath + "' INTO TABLE foo") -// if err != nil { -// ... -// +// filePath := "/home/gopher/data.csv" +// mysql.RegisterLocalFile(filePath) +// err := db.Exec("LOAD DATA LOCAL INFILE '" + filePath + "' INTO TABLE foo") +// if err != nil { +// ... func RegisterLocalFile(filePath string) { fileRegisterLock.Lock() // lazy map init if fileRegister == nil { - fileRegister = make(map[string]bool) + fileRegister = make(map[string]struct{}) } - fileRegister[strings.Trim(filePath, `"`)] = true + fileRegister[strings.Trim(filePath, `"`)] = struct{}{} fileRegisterLock.Unlock() } @@ -58,15 +57,14 @@ func DeregisterLocalFile(filePath string) { // If the handler returns a io.ReadCloser Close() is called when the // request is finished. // -// mysql.RegisterReaderHandler("data", func() io.Reader { -// var csvReader io.Reader // Some Reader that returns CSV data -// ... // Open Reader here -// return csvReader -// }) -// err := db.Exec("LOAD DATA LOCAL INFILE 'Reader::data' INTO TABLE foo") -// if err != nil { -// ... -// +// mysql.RegisterReaderHandler("data", func() io.Reader { +// var csvReader io.Reader // Some Reader that returns CSV data +// ... // Open Reader here +// return csvReader +// }) +// err := db.Exec("LOAD DATA LOCAL INFILE 'Reader::data' INTO TABLE foo") +// if err != nil { +// ... func RegisterReaderHandler(name string, handler func() io.Reader) { readerRegisterLock.Lock() // lazy map init @@ -93,10 +91,11 @@ func deferredClose(err *error, closer io.Closer) { } } -func (mc *mysqlConn) handleInFileRequest(name string) (err error) { +const defaultPacketSize = 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + +func (mc *okHandler) handleInFileRequest(name string) (err error) { var rdr io.Reader - var data []byte - packetSize := 16 * 1024 // 16KB is small enough for disk readahead and large enough for TCP + packetSize := defaultPacketSize if mc.maxWriteSize < packetSize { packetSize = mc.maxWriteSize } @@ -116,17 +115,17 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { defer deferredClose(&err, cl) } } else { - err = fmt.Errorf("Reader '%s' is ", name) + err = fmt.Errorf("reader '%s' is ", name) } } else { - err = fmt.Errorf("Reader '%s' is not registered", name) + err = fmt.Errorf("reader '%s' is not registered", name) } } else { // File name = strings.Trim(name, `"`) fileRegisterLock.RLock() - fr := fileRegister[name] + _, exists := fileRegister[name] fileRegisterLock.RUnlock() - if mc.cfg.AllowAllFiles || fr { + if mc.cfg.AllowAllFiles || exists { var file *os.File var fi os.FileInfo @@ -147,14 +146,16 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { } // send content packets + var data []byte + // if packetSize == 0, the Reader contains no data if err == nil && packetSize > 0 { - data := make([]byte, 4+packetSize) + data = make([]byte, 4+packetSize) var n int for err == nil { n, err = rdr.Read(data[4:]) if n > 0 { - if ioErr := mc.writePacket(data[:4+n]); ioErr != nil { + if ioErr := mc.conn().writePacket(data[:4+n]); ioErr != nil { return ioErr } } @@ -168,15 +169,16 @@ func (mc *mysqlConn) handleInFileRequest(name string) (err error) { if data == nil { data = make([]byte, 4) } - if ioErr := mc.writePacket(data[:4]); ioErr != nil { + if ioErr := mc.conn().writePacket(data[:4]); ioErr != nil { return ioErr } + mc.conn().syncSequence() // read OK packet if err == nil { return mc.readResultOK() } - mc.readPacket() + mc.conn().readPacket() return err } diff --git a/nulltime.go b/nulltime.go index 17af92ddc..316a48aae 100644 --- a/nulltime.go +++ b/nulltime.go @@ -19,16 +19,16 @@ import ( // NullTime implements the Scanner interface so // it can be used as a scan destination: // -// var nt NullTime -// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) -// ... -// if nt.Valid { -// // use nt.Time -// } else { -// // NULL value -// } +// var nt NullTime +// err := db.QueryRow("SELECT time FROM foo WHERE id=?", id).Scan(&nt) +// ... +// if nt.Valid { +// // use nt.Time +// } else { +// // NULL value +// } // -// This NullTime implementation is not driver-specific +// # This NullTime implementation is not driver-specific // // Deprecated: NullTime doesn't honor the loc DSN parameter. // NullTime.Scan interprets a time as UTC, not the loc DSN parameter. @@ -38,7 +38,7 @@ type NullTime sql.NullTime // Scan implements the Scanner interface. // The value type must be time.Time or string / []byte (formatted time-string), // otherwise Scan fails. -func (nt *NullTime) Scan(value interface{}) (err error) { +func (nt *NullTime) Scan(value any) (err error) { if value == nil { nt.Time, nt.Valid = time.Time{}, false return @@ -59,7 +59,7 @@ func (nt *NullTime) Scan(value interface{}) (err error) { } nt.Valid = false - return fmt.Errorf("Can't convert %T to time.Time", value) + return fmt.Errorf("can't convert %T to time.Time", value) } // Value implements the driver Valuer interface. diff --git a/nulltime_test.go b/nulltime_test.go index a14ec0607..4f1d9029e 100644 --- a/nulltime_test.go +++ b/nulltime_test.go @@ -23,7 +23,7 @@ var ( func TestScanNullTime(t *testing.T) { var scanTests = []struct { - in interface{} + in any error bool valid bool time time.Time diff --git a/packets.go b/packets.go index 1867ecab2..4b8362160 100644 --- a/packets.go +++ b/packets.go @@ -14,75 +14,102 @@ import ( "database/sql/driver" "encoding/binary" "encoding/json" - "errors" "fmt" "io" "math" + "strconv" "time" ) -// Packets documentation: -// http://dev.mysql.com/doc/internals/en/client-server-protocol.html +// MySQL client/server protocol documentations. +// https://dev.mysql.com/doc/dev/mysql-server/latest/PAGE_PROTOCOL.html +// https://mariadb.com/kb/en/clientserver-protocol/ // Read packet to buffer 'data' func (mc *mysqlConn) readPacket() ([]byte, error) { var prevData []byte + invalidSequence := false + + readNext := mc.buf.readNext + if mc.compress { + readNext = mc.compIO.readNext + } + for { // read packet header - data, err := mc.buf.readNext(4) + data, err := readNext(4, mc.readWithTimeout) if err != nil { + mc.close() if cerr := mc.canceled.Value(); cerr != nil { return nil, cerr } - errLog.Print(err) - mc.Close() + mc.log(err) return nil, ErrInvalidConn } // packet length [24 bit] - pktLen := int(uint32(data[0]) | uint32(data[1])<<8 | uint32(data[2])<<16) - - // check packet sync [8 bit] - if data[3] != mc.sequence { - if data[3] > mc.sequence { - return nil, ErrPktSyncMul + pktLen := getUint24(data[:3]) + seq := data[3] + + if mc.compress { + // MySQL and MariaDB doesn't check packet nr in compressed packet. + if debug && seq != mc.compressSequence { + fmt.Printf("[debug] mismatched compression sequence nr: expected: %v, got %v", + mc.compressSequence, seq) } - return nil, ErrPktSync + mc.compressSequence = seq + 1 + } else { + // check packet sync [8 bit] + if seq != mc.sequence { + mc.log(fmt.Sprintf("[warn] unexpected seq nr: expected %v, got %v", mc.sequence, seq)) + // For large packets, we stop reading as soon as sync error. + if len(prevData) > 0 { + mc.close() + return nil, ErrPktSyncMul + } + invalidSequence = true + } + mc.sequence++ } - mc.sequence++ // packets with length 0 terminate a previous packet which is a // multiple of (2^24)-1 bytes long if pktLen == 0 { // there was no previous packet if prevData == nil { - errLog.Print(ErrMalformPkt) - mc.Close() + mc.log(ErrMalformPkt) + mc.close() return nil, ErrInvalidConn } - return prevData, nil } // read packet body [pktLen bytes] - data, err = mc.buf.readNext(pktLen) + data, err = readNext(pktLen, mc.readWithTimeout) if err != nil { + mc.close() if cerr := mc.canceled.Value(); cerr != nil { return nil, cerr } - errLog.Print(err) - mc.Close() + mc.log(err) return nil, ErrInvalidConn } // return data if this was the last packet if pktLen < maxPacketSize { // zero allocations for non-split packets - if prevData == nil { - return data, nil + if prevData != nil { + data = append(prevData, data...) } - - return append(prevData, data...), nil + if invalidSequence { + mc.close() + // return sync error only for regular packet. + // error packets may have wrong sequence number. + if data[0] != iERR { + return nil, ErrPktSync + } + } + return data, nil } prevData = append(prevData, data...) @@ -92,89 +119,52 @@ func (mc *mysqlConn) readPacket() ([]byte, error) { // Write packet buffer 'data' func (mc *mysqlConn) writePacket(data []byte) error { pktLen := len(data) - 4 - if pktLen > mc.maxAllowedPacket { return ErrPktTooLarge } - // Perform a stale connection check. We only perform this check for - // the first query on a connection that has been checked out of the - // connection pool: a fresh connection from the pool is more likely - // to be stale, and it has not performed any previous writes that - // could cause data corruption, so it's safe to return ErrBadConn - // if the check fails. - if mc.reset { - mc.reset = false - conn := mc.netConn - if mc.rawConn != nil { - conn = mc.rawConn - } - var err error - // If this connection has a ReadTimeout which we've been setting on - // reads, reset it to its default value before we attempt a non-blocking - // read, otherwise the scheduler will just time us out before we can read - if mc.cfg.ReadTimeout != 0 { - err = conn.SetReadDeadline(time.Time{}) - } - if err == nil && mc.cfg.CheckConnLiveness { - err = connCheck(conn) - } - if err != nil { - errLog.Print("closing bad idle connection: ", err) - mc.Close() - return driver.ErrBadConn - } + writeFunc := mc.writeWithTimeout + if mc.compress { + writeFunc = mc.compIO.writePackets } for { - var size int - if pktLen >= maxPacketSize { - data[0] = 0xff - data[1] = 0xff - data[2] = 0xff - size = maxPacketSize - } else { - data[0] = byte(pktLen) - data[1] = byte(pktLen >> 8) - data[2] = byte(pktLen >> 16) - size = pktLen - } + size := min(maxPacketSize, pktLen) + putUint24(data[:3], size) data[3] = mc.sequence // Write packet - if mc.writeTimeout > 0 { - if err := mc.netConn.SetWriteDeadline(time.Now().Add(mc.writeTimeout)); err != nil { - return err - } - } - - n, err := mc.netConn.Write(data[:4+size]) - if err == nil && n == 4+size { - mc.sequence++ - if size != maxPacketSize { - return nil - } - pktLen -= size - data = data[size:] - continue + if debug { + fmt.Printf("writePacket: size=%v seq=%v", size, mc.sequence) } - // Handle error - if err == nil { // n != len(data) + n, err := writeFunc(data[:4+size]) + if err != nil { mc.cleanup() - errLog.Print(ErrMalformPkt) - } else { if cerr := mc.canceled.Value(); cerr != nil { return cerr } if n == 0 && pktLen == len(data)-4 { // only for the first loop iteration when nothing was written yet + mc.log(err) return errBadConnNoWrite + } else { + return err } + } + if n != 4+size { + // io.Writer(b) must return a non-nil error if it cannot write len(b) bytes. + // The io.ErrShortWrite error is used to indicate that this rule has not been followed. mc.cleanup() - errLog.Print(err) + return io.ErrShortWrite + } + + mc.sequence++ + if size != maxPacketSize { + return nil } - return ErrInvalidConn + pktLen -= size + data = data[size:] } } @@ -187,11 +177,6 @@ func (mc *mysqlConn) writePacket(data []byte) error { func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err error) { data, err = mc.readPacket() if err != nil { - // for init we can rewrite this to ErrBadConn for sql.Driver to retry, since - // in connection initialization we don't risk retrying non-idempotent actions. - if err == ErrInvalidConn { - return nil, "", driver.ErrBadConn - } return } @@ -223,9 +208,9 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro if mc.flags&clientProtocol41 == 0 { return nil, "", ErrOldProtocol } - if mc.flags&clientSSL == 0 && mc.cfg.tls != nil { - if mc.cfg.TLSConfig == "preferred" { - mc.cfg.tls = nil + if mc.flags&clientSSL == 0 && mc.cfg.TLS != nil { + if mc.cfg.AllowFallbackToPlaintext { + mc.cfg.TLS = nil } else { return nil, "", ErrNoTLS } @@ -235,12 +220,15 @@ func (mc *mysqlConn) readHandshakePacket() (data []byte, plugin string, err erro if len(data) > pos { // character set [1 byte] // status flags [2 bytes] + pos += 3 // capability flags (upper 2 bytes) [2 bytes] + mc.flags |= clientFlag(binary.LittleEndian.Uint16(data[pos:pos+2])) << 16 + pos += 2 // length of auth-plugin-data [1 byte] // reserved (all [00]) [10 bytes] - pos += 1 + 2 + 2 + 1 + 10 + pos += 11 - // second part of the password cipher [mininum 13 bytes], + // second part of the password cipher [minimum 13 bytes], // where len=MAX(13, length of auth-plugin-data - 8) // // The web documentation is ambiguous about the length. However, @@ -286,14 +274,19 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string clientLocalFiles | clientPluginAuth | clientMultiResults | + mc.flags&clientConnectAttrs | mc.flags&clientLongFlag + sendConnectAttrs := mc.flags&clientConnectAttrs != 0 + if mc.cfg.ClientFoundRows { clientFlags |= clientFoundRows } - + if mc.cfg.compress && mc.flags&clientCompress == clientCompress { + clientFlags |= clientCompress + } // To enable TLS / SSL - if mc.cfg.tls != nil { + if mc.cfg.TLS != nil { clientFlags |= clientSSL } @@ -319,34 +312,38 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string pktLen += n + 1 } + // encode length of the connection attributes + var connAttrsLEI []byte + if sendConnectAttrs { + var connAttrsLEIBuf [9]byte + connAttrsLen := len(mc.connector.encodedAttributes) + connAttrsLEI = appendLengthEncodedInteger(connAttrsLEIBuf[:0], uint64(connAttrsLen)) + pktLen += len(connAttrsLEI) + len(mc.connector.encodedAttributes) + } + // Calculate packet length and get buffer with that size - data, err := mc.buf.takeSmallBuffer(pktLen + 4) + data, err := mc.buf.takeBuffer(pktLen + 4) if err != nil { - // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) - return errBadConnNoWrite + mc.cleanup() + return err } // ClientFlags [32 bit] - data[4] = byte(clientFlags) - data[5] = byte(clientFlags >> 8) - data[6] = byte(clientFlags >> 16) - data[7] = byte(clientFlags >> 24) + binary.LittleEndian.PutUint32(data[4:], uint32(clientFlags)) // MaxPacketSize [32 bit] (none) - data[8] = 0x00 - data[9] = 0x00 - data[10] = 0x00 - data[11] = 0x00 - - // Charset [1 byte] - var found bool - data[12], found = collations[mc.cfg.Collation] - if !found { - // Note possibility for false negatives: - // could be triggered although the collation is valid if the - // collations map does not contain entries the server supports. - return errors.New("unknown collation") + binary.LittleEndian.PutUint32(data[8:], 0) + + // Collation ID [1 byte] + data[12] = defaultCollationID + if cname := mc.cfg.Collation; cname != "" { + colID, ok := collations[cname] + if ok { + data[12] = colID + } else if len(mc.cfg.charsets) > 0 { + // When cfg.charset is set, the collation is set by `SET NAMES COLLATE `. + return fmt.Errorf("unknown collation: %q", cname) + } } // Filler [23 bytes] (all 0x00) @@ -357,20 +354,21 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string // SSL Connection Request Packet // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::SSLRequest - if mc.cfg.tls != nil { + if mc.cfg.TLS != nil { // Send TLS / SSL request packet if err := mc.writePacket(data[:(4+4+1+23)+4]); err != nil { return err } // Switch to TLS - tlsConn := tls.Client(mc.netConn, mc.cfg.tls) + tlsConn := tls.Client(mc.netConn, mc.cfg.TLS) if err := tlsConn.Handshake(); err != nil { + if cerr := mc.canceled.Value(); cerr != nil { + return cerr + } return err } - mc.rawConn = mc.netConn mc.netConn = tlsConn - mc.buf.nc = tlsConn } // User [null terminated string] @@ -395,6 +393,12 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string data[pos] = 0x00 pos++ + // Connection Attributes + if sendConnectAttrs { + pos += copy(data[pos:], connAttrsLEI) + pos += copy(data[pos:], []byte(mc.connector.encodedAttributes)) + } + // Send Auth packet return mc.writePacket(data[:pos]) } @@ -402,11 +406,10 @@ func (mc *mysqlConn) writeHandshakeResponsePacket(authResp []byte, plugin string // http://dev.mysql.com/doc/internals/en/connection-phase-packets.html#packet-Protocol::AuthSwitchResponse func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { pktLen := 4 + len(authData) - data, err := mc.buf.takeSmallBuffer(pktLen) + data, err := mc.buf.takeBuffer(pktLen) if err != nil { - // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) - return errBadConnNoWrite + mc.cleanup() + return err } // Add the auth data [EOF] @@ -420,13 +423,11 @@ func (mc *mysqlConn) writeAuthSwitchPacket(authData []byte) error { func (mc *mysqlConn) writeCommandPacket(command byte) error { // Reset Packet Sequence - mc.sequence = 0 + mc.resetSequence() data, err := mc.buf.takeSmallBuffer(4 + 1) if err != nil { - // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) - return errBadConnNoWrite + return err } // Add command byte @@ -438,14 +439,12 @@ func (mc *mysqlConn) writeCommandPacket(command byte) error { func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { // Reset Packet Sequence - mc.sequence = 0 + mc.resetSequence() pktLen := 1 + len(arg) data, err := mc.buf.takeBuffer(pktLen + 4) if err != nil { - // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) - return errBadConnNoWrite + return err } // Add command byte @@ -455,28 +454,25 @@ func (mc *mysqlConn) writeCommandPacketStr(command byte, arg string) error { copy(data[5:], arg) // Send CMD packet - return mc.writePacket(data) + err = mc.writePacket(data) + mc.syncSequence() + return err } func (mc *mysqlConn) writeCommandPacketUint32(command byte, arg uint32) error { // Reset Packet Sequence - mc.sequence = 0 + mc.resetSequence() data, err := mc.buf.takeSmallBuffer(4 + 1 + 4) if err != nil { - // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) - return errBadConnNoWrite + return err } // Add command byte data[4] = command // Add arg [32 bit] - data[5] = byte(arg) - data[6] = byte(arg >> 8) - data[7] = byte(arg >> 16) - data[8] = byte(arg >> 24) + binary.LittleEndian.PutUint32(data[5:], arg) // Send CMD packet return mc.writePacket(data) @@ -496,7 +492,9 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { switch data[0] { case iOK: - return nil, "", mc.handleOkPacket(data) + // resultUnchanged, since auth happens before any queries or + // commands have been executed. + return nil, "", mc.resultUnchanged().handleOkPacket(data) case iAuthMoreData: return data[1:], "", err @@ -512,6 +510,9 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { } plugin := string(data[1:pluginEndIndex]) authData := data[pluginEndIndex+1:] + if len(authData) > 0 && authData[len(authData)-1] == 0 { + authData = authData[:len(authData)-1] + } return authData, plugin, nil default: // Error otherwise @@ -519,9 +520,9 @@ func (mc *mysqlConn) readAuthResult() ([]byte, string, error) { } } -// Returns error if Packet is not an 'Result OK'-Packet -func (mc *mysqlConn) readResultOK() error { - data, err := mc.readPacket() +// Returns error if Packet is not a 'Result OK'-Packet +func (mc *okHandler) readResultOK() error { + data, err := mc.conn().readPacket() if err != nil { return err } @@ -529,35 +530,37 @@ func (mc *mysqlConn) readResultOK() error { if data[0] == iOK { return mc.handleOkPacket(data) } - return mc.handleErrorPacket(data) + return mc.conn().handleErrorPacket(data) } // Result Set Header Packet -// http://dev.mysql.com/doc/internals/en/com-query-response.html#packet-ProtocolText::Resultset -func (mc *mysqlConn) readResultSetHeaderPacket() (int, error) { - data, err := mc.readPacket() - if err == nil { - switch data[0] { +// https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response.html +func (mc *okHandler) readResultSetHeaderPacket() (int, error) { + // handleOkPacket replaces both values; other cases leave the values unchanged. + mc.result.affectedRows = append(mc.result.affectedRows, 0) + mc.result.insertIds = append(mc.result.insertIds, 0) - case iOK: - return 0, mc.handleOkPacket(data) - - case iERR: - return 0, mc.handleErrorPacket(data) + data, err := mc.conn().readPacket() + if err != nil { + return 0, err + } - case iLocalInFile: - return 0, mc.handleInFileRequest(string(data[1:])) - } + switch data[0] { + case iOK: + return 0, mc.handleOkPacket(data) - // column count - num, _, n := readLengthEncodedInteger(data) - if n-len(data) == 0 { - return int(num), nil - } + case iERR: + return 0, mc.conn().handleErrorPacket(data) - return 0, ErrMalformPkt + case iLocalInFile: + return 0, mc.handleInFileRequest(string(data[1:])) } - return 0, err + + // column count + // https://dev.mysql.com/doc/dev/mysql-server/latest/page_protocol_com_query_response_text_resultset.html + num, _, _ := readLengthEncodedInteger(data) + // ignore remaining data in the packet. see #1478. + return int(num), nil } // Error Packet @@ -574,7 +577,8 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { // 1792: ER_CANT_EXECUTE_IN_READ_ONLY_TRANSACTION // 1290: ER_OPTION_PREVENTS_STATEMENT (returned by Aurora during failover) - if (errno == 1792 || errno == 1290) && mc.cfg.RejectReadOnly { + // 1836: ER_READ_ONLY_MODE + if (errno == 1792 || errno == 1290 || errno == 1836) && mc.cfg.RejectReadOnly { // Oops; we are connected to a read-only connection, and won't be able // to issue any write statements. Since RejectReadOnly is configured, // we throw away this connection hoping this one would have write @@ -588,37 +592,81 @@ func (mc *mysqlConn) handleErrorPacket(data []byte) error { return driver.ErrBadConn } + me := &MySQLError{Number: errno} + pos := 3 // SQL State [optional: # + 5bytes string] if data[3] == 0x23 { - //sqlstate := string(data[4 : 4+5]) + copy(me.SQLState[:], data[4:4+5]) pos = 9 } // Error Message [string] - return &MySQLError{ - Number: errno, - Message: string(data[pos:]), - } + me.Message = string(data[pos:]) + + return me } func readStatus(b []byte) statusFlag { return statusFlag(b[0]) | statusFlag(b[1])<<8 } +// Returns an instance of okHandler for codepaths where mysqlConn.result doesn't +// need to be cleared first (e.g. during authentication, or while additional +// resultsets are being fetched.) +func (mc *mysqlConn) resultUnchanged() *okHandler { + return (*okHandler)(mc) +} + +// okHandler represents the state of the connection when mysqlConn.result has +// been prepared for processing of OK packets. +// +// To correctly populate mysqlConn.result (updated by handleOkPacket()), all +// callpaths must either: +// +// 1. first clear it using clearResult(), or +// 2. confirm that they don't need to (by calling resultUnchanged()). +// +// Both return an instance of type *okHandler. +type okHandler mysqlConn + +// Exposes the underlying type's methods. +func (mc *okHandler) conn() *mysqlConn { + return (*mysqlConn)(mc) +} + +// clearResult clears the connection's stored affectedRows and insertIds +// fields. +// +// It returns a handler that can process OK responses. +func (mc *mysqlConn) clearResult() *okHandler { + mc.result = mysqlResult{} + return (*okHandler)(mc) +} + // Ok Packet // http://dev.mysql.com/doc/internals/en/generic-response-packets.html#packet-OK_Packet -func (mc *mysqlConn) handleOkPacket(data []byte) error { +func (mc *okHandler) handleOkPacket(data []byte) error { var n, m int + var affectedRows, insertId uint64 // 0x00 [1 byte] // Affected rows [Length Coded Binary] - mc.affectedRows, _, n = readLengthEncodedInteger(data[1:]) + affectedRows, _, n = readLengthEncodedInteger(data[1:]) // Insert id [Length Coded Binary] - mc.insertId, _, m = readLengthEncodedInteger(data[1+n:]) + insertId, _, m = readLengthEncodedInteger(data[1+n:]) + + // Update for the current statement result (only used by + // readResultSetHeaderPacket). + if len(mc.result.affectedRows) > 0 { + mc.result.affectedRows[len(mc.result.affectedRows)-1] = int64(affectedRows) + } + if len(mc.result.insertIds) > 0 { + mc.result.insertIds[len(mc.result.insertIds)-1] = int64(insertId) + } // server_status [2 bytes] mc.status = readStatus(data[1+n+m : 1+n+m+2]) @@ -769,7 +817,8 @@ func (rows *textRows) readRow(dest []driver.Value) error { for i := range dest { // Read bytes and convert to string - dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) + var buf []byte + buf, isNull, n, err = readLengthEncodedString(data[pos:]) pos += n if err != nil { @@ -781,19 +830,40 @@ func (rows *textRows) readRow(dest []driver.Value) error { continue } - if !mc.parseTime { - continue - } - - // Parse time field switch rows.rs.columns[i].fieldType { case fieldTypeTimestamp, fieldTypeDateTime, fieldTypeDate, fieldTypeNewDate: - if dest[i], err = parseDateTime(dest[i].([]byte), mc.cfg.Loc); err != nil { - return err + if mc.parseTime { + dest[i], err = parseDateTime(buf, mc.cfg.Loc) + } else { + dest[i] = buf } + + case fieldTypeTiny, fieldTypeShort, fieldTypeInt24, fieldTypeYear, fieldTypeLong: + dest[i], err = strconv.ParseInt(string(buf), 10, 64) + + case fieldTypeLongLong: + if rows.rs.columns[i].flags&flagUnsigned != 0 { + dest[i], err = strconv.ParseUint(string(buf), 10, 64) + } else { + dest[i], err = strconv.ParseInt(string(buf), 10, 64) + } + + case fieldTypeFloat: + var d float64 + d, err = strconv.ParseFloat(string(buf), 32) + dest[i] = float32(d) + + case fieldTypeDouble: + dest[i], err = strconv.ParseFloat(string(buf), 64) + + default: + dest[i] = buf + } + if err != nil { + return err } } @@ -875,19 +945,15 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { pktLen = dataOffset + argLen } - stmt.mc.sequence = 0 + stmt.mc.resetSequence() // Add command byte [1 byte] data[4] = comStmtSendLongData // Add stmtID [32 bit] - data[5] = byte(stmt.id) - data[6] = byte(stmt.id >> 8) - data[7] = byte(stmt.id >> 16) - data[8] = byte(stmt.id >> 24) + binary.LittleEndian.PutUint32(data[5:], stmt.id) // Add paramID [16 bit] - data[9] = byte(paramID) - data[10] = byte(paramID >> 8) + binary.LittleEndian.PutUint16(data[9:], uint16(paramID)) // Send CMD packet err := stmt.mc.writePacket(data[:4+pktLen]) @@ -896,11 +962,10 @@ func (stmt *mysqlStmt) writeCommandLongData(paramID int, arg []byte) error { continue } return err - } // Reset Packet Sequence - stmt.mc.sequence = 0 + stmt.mc.resetSequence() return nil } @@ -925,7 +990,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { } // Reset packet-sequence - mc.sequence = 0 + mc.resetSequence() var data []byte var err error @@ -937,28 +1002,20 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In this case the len(data) == cap(data) which is used to optimise the flow below. } if err != nil { - // cannot take the buffer. Something must be wrong with the connection - errLog.Print(err) - return errBadConnNoWrite + return err } // command [1 byte] data[4] = comStmtExecute // statement_id [4 bytes] - data[5] = byte(stmt.id) - data[6] = byte(stmt.id >> 8) - data[7] = byte(stmt.id >> 16) - data[8] = byte(stmt.id >> 24) + binary.LittleEndian.PutUint32(data[5:], stmt.id) // flags (0: CURSOR_TYPE_NO_CURSOR) [1 byte] data[9] = 0x00 // iteration_count (uint32(1)) [4 bytes] - data[10] = 0x01 - data[11] = 0x00 - data[12] = 0x00 - data[13] = 0x00 + binary.LittleEndian.PutUint32(data[10:], 1) if len(args) > 0 { pos := minPktLen @@ -1012,50 +1069,17 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { case int64: paramTypes[i+i] = byte(fieldTypeLongLong) paramTypes[i+i+1] = 0x00 - - if cap(paramValues)-len(paramValues)-8 >= 0 { - paramValues = paramValues[:len(paramValues)+8] - binary.LittleEndian.PutUint64( - paramValues[len(paramValues)-8:], - uint64(v), - ) - } else { - paramValues = append(paramValues, - uint64ToBytes(uint64(v))..., - ) - } + paramValues = binary.LittleEndian.AppendUint64(paramValues, uint64(v)) case uint64: paramTypes[i+i] = byte(fieldTypeLongLong) paramTypes[i+i+1] = 0x80 // type is unsigned - - if cap(paramValues)-len(paramValues)-8 >= 0 { - paramValues = paramValues[:len(paramValues)+8] - binary.LittleEndian.PutUint64( - paramValues[len(paramValues)-8:], - uint64(v), - ) - } else { - paramValues = append(paramValues, - uint64ToBytes(uint64(v))..., - ) - } + paramValues = binary.LittleEndian.AppendUint64(paramValues, uint64(v)) case float64: paramTypes[i+i] = byte(fieldTypeDouble) paramTypes[i+i+1] = 0x00 - - if cap(paramValues)-len(paramValues)-8 >= 0 { - paramValues = paramValues[:len(paramValues)+8] - binary.LittleEndian.PutUint64( - paramValues[len(paramValues)-8:], - math.Float64bits(v), - ) - } else { - paramValues = append(paramValues, - uint64ToBytes(math.Float64bits(v))..., - ) - } + paramValues = binary.LittleEndian.AppendUint64(paramValues, math.Float64bits(v)) case bool: paramTypes[i+i] = byte(fieldTypeTiny) @@ -1116,7 +1140,7 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { if v.IsZero() { b = append(b, "0000-00-00"...) } else { - b, err = appendDateTime(b, v.In(mc.cfg.Loc)) + b, err = appendDateTime(b, v.In(mc.cfg.Loc), mc.cfg.timeTruncate) if err != nil { return err } @@ -1136,20 +1160,21 @@ func (stmt *mysqlStmt) writeExecutePacket(args []driver.Value) error { // In that case we must build the data packet with the new values buffer if valuesCap != cap(paramValues) { data = append(data[:pos], paramValues...) - if err = mc.buf.store(data); err != nil { - errLog.Print(err) - return errBadConnNoWrite - } + mc.buf.store(data) // allow this buffer to be reused } pos += len(paramValues) data = data[:pos] } - return mc.writePacket(data) + err = mc.writePacket(data) + mc.syncSequence() + return err } -func (mc *mysqlConn) discardResults() error { +// For each remaining resultset in the stream, discards its rows and updates +// mc.affectedRows and mc.insertIds. +func (mc *okHandler) discardResults() error { for mc.status&statusMoreResultsExists != 0 { resLen, err := mc.readResultSetHeaderPacket() if err != nil { @@ -1157,11 +1182,11 @@ func (mc *mysqlConn) discardResults() error { } if resLen > 0 { // columns - if err := mc.readUntilEOF(); err != nil { + if err := mc.conn().readUntilEOF(); err != nil { return err } // rows - if err := mc.readUntilEOF(); err != nil { + if err := mc.conn().readUntilEOF(); err != nil { return err } } @@ -1268,7 +1293,8 @@ func (rows *binaryRows) readRow(dest []driver.Value) error { case fieldTypeDecimal, fieldTypeNewDecimal, fieldTypeVarChar, fieldTypeBit, fieldTypeEnum, fieldTypeSet, fieldTypeTinyBLOB, fieldTypeMediumBLOB, fieldTypeLongBLOB, fieldTypeBLOB, - fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON: + fieldTypeVarString, fieldTypeString, fieldTypeGeometry, fieldTypeJSON, + fieldTypeVector: var isNull bool var n int dest[i], isNull, n, err = readLengthEncodedString(data[pos:]) diff --git a/packets_test.go b/packets_test.go index b61e4dbf7..694b0564c 100644 --- a/packets_test.go +++ b/packets_test.go @@ -96,9 +96,11 @@ var _ net.Conn = new(mockConn) func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { conn := new(mockConn) + connector := newConnector(NewConfig()) mc := &mysqlConn{ - buf: newBuffer(conn), - cfg: NewConfig(), + buf: newBuffer(), + cfg: connector.cfg, + connector: connector, netConn: conn, closech: make(chan struct{}), maxAllowedPacket: defaultMaxAllowedPacket, @@ -110,7 +112,9 @@ func newRWMockConn(sequence uint8) (*mockConn, *mysqlConn) { func TestReadPacketSingleByte(t *testing.T) { conn := new(mockConn) mc := &mysqlConn{ - buf: newBuffer(conn), + netConn: conn, + buf: newBuffer(), + cfg: NewConfig(), } conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} @@ -128,37 +132,43 @@ func TestReadPacketSingleByte(t *testing.T) { } func TestReadPacketWrongSequenceID(t *testing.T) { - conn := new(mockConn) - mc := &mysqlConn{ - buf: newBuffer(conn), - } - - // too low sequence id - conn.data = []byte{0x01, 0x00, 0x00, 0x00, 0xff} - conn.maxReads = 1 - mc.sequence = 1 - _, err := mc.readPacket() - if err != ErrPktSync { - t.Errorf("expected ErrPktSync, got %v", err) - } - - // reset - conn.reads = 0 - mc.sequence = 0 - mc.buf = newBuffer(conn) - - // too high sequence id - conn.data = []byte{0x01, 0x00, 0x00, 0x42, 0xff} - _, err = mc.readPacket() - if err != ErrPktSyncMul { - t.Errorf("expected ErrPktSyncMul, got %v", err) + for _, testCase := range []struct { + ClientSequenceID byte + ServerSequenceID byte + ExpectedErr error + }{ + { + ClientSequenceID: 1, + ServerSequenceID: 0, + ExpectedErr: ErrPktSync, + }, + { + ClientSequenceID: 0, + ServerSequenceID: 0x42, + ExpectedErr: ErrPktSync, + }, + } { + conn, mc := newRWMockConn(testCase.ClientSequenceID) + + conn.data = []byte{0x01, 0x00, 0x00, testCase.ServerSequenceID, 0x22} + _, err := mc.readPacket() + if err != testCase.ExpectedErr { + t.Errorf("expected %v, got %v", testCase.ExpectedErr, err) + } + + // connection should not be returned to the pool in this state + if mc.IsValid() { + t.Errorf("expected IsValid() to be false") + } } } func TestReadPacketSplit(t *testing.T) { conn := new(mockConn) mc := &mysqlConn{ - buf: newBuffer(conn), + netConn: conn, + buf: newBuffer(), + cfg: NewConfig(), } data := make([]byte, maxPacketSize*2+4*3) @@ -179,7 +189,7 @@ func TestReadPacketSplit(t *testing.T) { data[4] = 0x11 data[maxPacketSize+3] = 0x22 - // 2nd packet has payload length 0 and squence id 1 + // 2nd packet has payload length 0 and sequence id 1 // 00 00 00 01 data[pkt2ofs+3] = 0x01 @@ -211,7 +221,7 @@ func TestReadPacketSplit(t *testing.T) { data[pkt2ofs+4] = 0x33 data[pkt2ofs+maxPacketSize+3] = 0x44 - // 3rd packet has payload length 0 and squence id 2 + // 3rd packet has payload length 0 and sequence id 2 // 00 00 00 02 data[pkt3ofs+3] = 0x02 @@ -263,8 +273,10 @@ func TestReadPacketSplit(t *testing.T) { func TestReadPacketFail(t *testing.T) { conn := new(mockConn) mc := &mysqlConn{ - buf: newBuffer(conn), + netConn: conn, + buf: newBuffer(), closech: make(chan struct{}), + cfg: NewConfig(), } // illegal empty (stand-alone) packet @@ -278,7 +290,7 @@ func TestReadPacketFail(t *testing.T) { // reset conn.reads = 0 mc.sequence = 0 - mc.buf = newBuffer(conn) + mc.buf = newBuffer() // fail to read header conn.closed = true @@ -291,7 +303,7 @@ func TestReadPacketFail(t *testing.T) { conn.closed = false conn.reads = 0 mc.sequence = 0 - mc.buf = newBuffer(conn) + mc.buf = newBuffer() // fail to read body conn.maxReads = 1 @@ -306,7 +318,8 @@ func TestReadPacketFail(t *testing.T) { func TestRegression801(t *testing.T) { conn := new(mockConn) mc := &mysqlConn{ - buf: newBuffer(conn), + netConn: conn, + buf: newBuffer(), cfg: new(Config), sequence: 42, closech: make(chan struct{}), diff --git a/result.go b/result.go index c6438d034..d51631468 100644 --- a/result.go +++ b/result.go @@ -8,15 +8,43 @@ package mysql +import "database/sql/driver" + +// Result exposes data not available through *connection.Result. +// +// This is accessible by executing statements using sql.Conn.Raw() and +// downcasting the returned result: +// +// res, err := rawConn.Exec(...) +// res.(mysql.Result).AllRowsAffected() +type Result interface { + driver.Result + // AllRowsAffected returns a slice containing the affected rows for each + // executed statement. + AllRowsAffected() []int64 + // AllLastInsertIds returns a slice containing the last inserted ID for each + // executed statement. + AllLastInsertIds() []int64 +} + type mysqlResult struct { - affectedRows int64 - insertId int64 + // One entry in both slices is created for every executed statement result. + affectedRows []int64 + insertIds []int64 } func (res *mysqlResult) LastInsertId() (int64, error) { - return res.insertId, nil + return res.insertIds[len(res.insertIds)-1], nil } func (res *mysqlResult) RowsAffected() (int64, error) { - return res.affectedRows, nil + return res.affectedRows[len(res.affectedRows)-1], nil +} + +func (res *mysqlResult) AllLastInsertIds() []int64 { + return append([]int64{}, res.insertIds...) // defensive copy +} + +func (res *mysqlResult) AllRowsAffected() []int64 { + return append([]int64{}, res.affectedRows...) // defensive copy } diff --git a/rows.go b/rows.go index 888bdb5f0..df98417b8 100644 --- a/rows.go +++ b/rows.go @@ -111,19 +111,13 @@ func (rows *mysqlRows) Close() (err error) { return err } - // flip the buffer for this connection if we need to drain it. - // note that for a successful query (i.e. one where rows.next() - // has been called until it returns false), `rows.mc` will be nil - // by the time the user calls `(*Rows).Close`, so we won't reach this - // see: https://github.com/golang/go/commit/651ddbdb5056ded455f47f9c494c67b389622a47 - mc.buf.flip() - // Remove unread packets from stream if !rows.rs.done { err = mc.readUntilEOF() } if err == nil { - if err = mc.discardResults(); err != nil { + handleOk := mc.clearResult() + if err = handleOk.discardResults(); err != nil { return err } } @@ -160,7 +154,15 @@ func (rows *mysqlRows) nextResultSet() (int, error) { return 0, io.EOF } rows.rs = resultSet{} - return rows.mc.readResultSetHeaderPacket() + // rows.mc.affectedRows and rows.mc.insertIds accumulate on each call to + // nextResultSet. + resLen, err := rows.mc.resultUnchanged().readResultSetHeaderPacket() + if err != nil { + // Clean up about multi-results flag + rows.rs.done = true + rows.mc.status = rows.mc.status & (^statusMoreResultsExists) + } + return resLen, err } func (rows *mysqlRows) nextNotEmptyResultSet() (int, error) { diff --git a/statement.go b/statement.go index 18a3ae498..35df85457 100644 --- a/statement.go +++ b/statement.go @@ -23,12 +23,13 @@ type mysqlStmt struct { } func (stmt *mysqlStmt) Close() error { - if stmt.mc == nil || stmt.mc.closed.IsSet() { - // driver.Stmt.Close can be called more than once, thus this function - // has to be idempotent. - // See also Issue #450 and golang/go#16019. - //errLog.Print(ErrInvalidConn) - return driver.ErrBadConn + if stmt.mc == nil || stmt.mc.closed.Load() { + // driver.Stmt.Close could be called more than once, thus this function + // had to be idempotent. See also Issue #450 and golang/go#16019. + // This bug has been fixed in Go 1.8. + // https://github.com/golang/go/commit/90b8a0ca2d0b565c7c7199ffcf77b15ea6b6db3a + // But we keep this function idempotent because it is safer. + return nil } err := stmt.mc.writeCommandPacketUint32(comStmtClose, stmt.id) @@ -50,8 +51,7 @@ func (stmt *mysqlStmt) CheckNamedValue(nv *driver.NamedValue) (err error) { } func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { - if stmt.mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + if stmt.mc.closed.Load() { return nil, driver.ErrBadConn } // Send command @@ -61,12 +61,10 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { } mc := stmt.mc - - mc.affectedRows = 0 - mc.insertId = 0 + handleOk := stmt.mc.clearResult() // Read Result - resLen, err := mc.readResultSetHeaderPacket() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return nil, err } @@ -83,14 +81,12 @@ func (stmt *mysqlStmt) Exec(args []driver.Value) (driver.Result, error) { } } - if err := mc.discardResults(); err != nil { + if err := handleOk.discardResults(); err != nil { return nil, err } - return &mysqlResult{ - affectedRows: int64(mc.affectedRows), - insertId: int64(mc.insertId), - }, nil + copied := mc.result + return &copied, nil } func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { @@ -98,8 +94,7 @@ func (stmt *mysqlStmt) Query(args []driver.Value) (driver.Rows, error) { } func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { - if stmt.mc.closed.IsSet() { - errLog.Print(ErrInvalidConn) + if stmt.mc.closed.Load() { return nil, driver.ErrBadConn } // Send command @@ -111,7 +106,8 @@ func (stmt *mysqlStmt) query(args []driver.Value) (*binaryRows, error) { mc := stmt.mc // Read Result - resLen, err := mc.readResultSetHeaderPacket() + handleOk := stmt.mc.clearResult() + resLen, err := handleOk.readResultSetHeaderPacket() if err != nil { return nil, err } @@ -144,7 +140,7 @@ type converter struct{} // implementation does not. This function should be kept in sync with // database/sql/driver defaultConverter.ConvertValue() except for that // deliberate difference. -func (c converter) ConvertValue(v interface{}) (driver.Value, error) { +func (c converter) ConvertValue(v any) (driver.Value, error) { if driver.IsValue(v) { return v, nil } @@ -157,7 +153,7 @@ func (c converter) ConvertValue(v interface{}) (driver.Value, error) { if driver.IsValue(sv) { return sv, nil } - // A value returend from the Valuer interface can be "a type handled by + // A value returned from the Valuer interface can be "a type handled by // a database driver's NamedValueChecker interface" so we should accept // uint64 here as well. if u, ok := sv.(uint64); ok { diff --git a/statement_test.go b/statement_test.go index 2563ece55..15f9d7c33 100644 --- a/statement_test.go +++ b/statement_test.go @@ -77,7 +77,7 @@ func TestConvertPointer(t *testing.T) { } func TestConvertSignedIntegers(t *testing.T) { - values := []interface{}{ + values := []any{ int8(-42), int16(-42), int32(-42), @@ -106,7 +106,7 @@ func (u myUint64) Value() (driver.Value, error) { } func TestConvertUnsignedIntegers(t *testing.T) { - values := []interface{}{ + values := []any{ uint8(42), uint16(42), uint32(42), diff --git a/transaction.go b/transaction.go index 417d72793..4a4b61001 100644 --- a/transaction.go +++ b/transaction.go @@ -13,7 +13,7 @@ type mysqlTx struct { } func (tx *mysqlTx) Commit() (err error) { - if tx.mc == nil || tx.mc.closed.IsSet() { + if tx.mc == nil || tx.mc.closed.Load() { return ErrInvalidConn } err = tx.mc.exec("COMMIT") @@ -22,7 +22,7 @@ func (tx *mysqlTx) Commit() (err error) { } func (tx *mysqlTx) Rollback() (err error) { - if tx.mc == nil || tx.mc.closed.IsSet() { + if tx.mc == nil || tx.mc.closed.Load() { return ErrInvalidConn } err = tx.mc.exec("ROLLBACK") diff --git a/utils.go b/utils.go index bcdee1b46..8716c26c5 100644 --- a/utils.go +++ b/utils.go @@ -35,26 +35,25 @@ var ( // Note: The provided tls.Config is exclusively owned by the driver after // registering it. // -// rootCertPool := x509.NewCertPool() -// pem, err := ioutil.ReadFile("/path/ca-cert.pem") -// if err != nil { -// log.Fatal(err) -// } -// if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { -// log.Fatal("Failed to append PEM.") -// } -// clientCert := make([]tls.Certificate, 0, 1) -// certs, err := tls.LoadX509KeyPair("/path/client-cert.pem", "/path/client-key.pem") -// if err != nil { -// log.Fatal(err) -// } -// clientCert = append(clientCert, certs) -// mysql.RegisterTLSConfig("custom", &tls.Config{ -// RootCAs: rootCertPool, -// Certificates: clientCert, -// }) -// db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") -// +// rootCertPool := x509.NewCertPool() +// pem, err := os.ReadFile("/path/ca-cert.pem") +// if err != nil { +// log.Fatal(err) +// } +// if ok := rootCertPool.AppendCertsFromPEM(pem); !ok { +// log.Fatal("Failed to append PEM.") +// } +// clientCert := make([]tls.Certificate, 0, 1) +// certs, err := tls.LoadX509KeyPair("/path/client-cert.pem", "/path/client-key.pem") +// if err != nil { +// log.Fatal(err) +// } +// clientCert = append(clientCert, certs) +// mysql.RegisterTLSConfig("custom", &tls.Config{ +// RootCAs: rootCertPool, +// Certificates: clientCert, +// }) +// db, err := sql.Open("mysql", "user@tcp(localhost:3306)/test?tls=custom") func RegisterTLSConfig(key string, config *tls.Config) error { if _, isBool := readBool(key); isBool || strings.ToLower(key) == "skip-verify" || strings.ToLower(key) == "preferred" { return fmt.Errorf("key '%s' is reserved", key) @@ -118,10 +117,6 @@ func parseDateTime(b []byte, loc *time.Location) (time.Time, error) { if err != nil { return time.Time{}, err } - if year <= 0 { - year = 1 - } - if b[4] != '-' { return time.Time{}, fmt.Errorf("bad value for field: `%c`", b[4]) } @@ -130,9 +125,6 @@ func parseDateTime(b []byte, loc *time.Location) (time.Time, error) { if err != nil { return time.Time{}, err } - if m <= 0 { - m = 1 - } month := time.Month(m) if b[7] != '-' { @@ -143,9 +135,6 @@ func parseDateTime(b []byte, loc *time.Location) (time.Time, error) { if err != nil { return time.Time{}, err } - if day <= 0 { - day = 1 - } if len(b) == 10 { return time.Date(year, month, day, 0, 0, 0, 0, loc), nil } @@ -276,7 +265,11 @@ func parseBinaryDateTime(num uint64, data []byte, loc *time.Location) (driver.Va return nil, fmt.Errorf("invalid DATETIME packet length %d", num) } -func appendDateTime(buf []byte, t time.Time) ([]byte, error) { +func appendDateTime(buf []byte, t time.Time, timeTruncate time.Duration) ([]byte, error) { + if timeTruncate > 0 { + t = t.Truncate(timeTruncate) + } + year, month, day := t.Date() hour, min, sec := t.Clock() nsec := t.Nanosecond() @@ -497,17 +490,16 @@ func formatBinaryTime(src []byte, length uint8) (driver.Value, error) { * Convert from and to bytes * ******************************************************************************/ -func uint64ToBytes(n uint64) []byte { - return []byte{ - byte(n), - byte(n >> 8), - byte(n >> 16), - byte(n >> 24), - byte(n >> 32), - byte(n >> 40), - byte(n >> 48), - byte(n >> 56), - } +// 24bit integer: used for packet headers. + +func putUint24(data []byte, n int) { + data[2] = byte(n >> 16) + data[1] = byte(n >> 8) + data[0] = byte(n) +} + +func getUint24(data []byte) int { + return int(data[2])<<16 | int(data[1])<<8 | int(data[0]) } func uint64ToString(n uint64) []byte { @@ -532,17 +524,7 @@ func uint64ToString(n uint64) []byte { return a[i:] } -// treats string value as unsigned integer representation -func stringToInt(b []byte) int { - val := 0 - for i := range b { - val *= 10 - val += int(b[i] - 0x30) - } - return val -} - -// returns the string read as a bytes slice, wheter the value is NULL, +// returns the string read as a bytes slice, whether the value is NULL, // the number of bytes read and an error, in case the string is longer than // the input slice func readLengthEncodedString(b []byte) ([]byte, bool, int, error) { @@ -593,18 +575,15 @@ func readLengthEncodedInteger(b []byte) (uint64, bool, int) { // 252: value of following 2 case 0xfc: - return uint64(b[1]) | uint64(b[2])<<8, false, 3 + return uint64(binary.LittleEndian.Uint16(b[1:])), false, 3 // 253: value of following 3 case 0xfd: - return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16, false, 4 + return uint64(getUint24(b[1:])), false, 4 // 254: value of following 8 case 0xfe: - return uint64(b[1]) | uint64(b[2])<<8 | uint64(b[3])<<16 | - uint64(b[4])<<24 | uint64(b[5])<<32 | uint64(b[6])<<40 | - uint64(b[7])<<48 | uint64(b[8])<<56, - false, 9 + return uint64(binary.LittleEndian.Uint64(b[1:])), false, 9 } // 0-250: value of first byte @@ -618,13 +597,19 @@ func appendLengthEncodedInteger(b []byte, n uint64) []byte { return append(b, byte(n)) case n <= 0xffff: - return append(b, 0xfc, byte(n), byte(n>>8)) + b = append(b, 0xfc) + return binary.LittleEndian.AppendUint16(b, uint16(n)) case n <= 0xffffff: return append(b, 0xfd, byte(n), byte(n>>8), byte(n>>16)) } - return append(b, 0xfe, byte(n), byte(n>>8), byte(n>>16), byte(n>>24), - byte(n>>32), byte(n>>40), byte(n>>48), byte(n>>56)) + b = append(b, 0xfe) + return binary.LittleEndian.AppendUint64(b, n) +} + +func appendLengthEncodedString(b []byte, s string) []byte { + b = appendLengthEncodedInteger(b, uint64(len(s))) + return append(b, s...) } // reserveBuffer checks cap(buf) and expand buffer to len(buf) + appendSize. @@ -652,32 +637,32 @@ func escapeBytesBackslash(buf, v []byte) []byte { for _, c := range v { switch c { case '\x00': - buf[pos] = '\\' buf[pos+1] = '0' + buf[pos] = '\\' pos += 2 case '\n': - buf[pos] = '\\' buf[pos+1] = 'n' + buf[pos] = '\\' pos += 2 case '\r': - buf[pos] = '\\' buf[pos+1] = 'r' + buf[pos] = '\\' pos += 2 case '\x1a': - buf[pos] = '\\' buf[pos+1] = 'Z' + buf[pos] = '\\' pos += 2 case '\'': - buf[pos] = '\\' buf[pos+1] = '\'' + buf[pos] = '\\' pos += 2 case '"': - buf[pos] = '\\' buf[pos+1] = '"' + buf[pos] = '\\' pos += 2 case '\\': - buf[pos] = '\\' buf[pos+1] = '\\' + buf[pos] = '\\' pos += 2 default: buf[pos] = c @@ -697,32 +682,32 @@ func escapeStringBackslash(buf []byte, v string) []byte { c := v[i] switch c { case '\x00': - buf[pos] = '\\' buf[pos+1] = '0' + buf[pos] = '\\' pos += 2 case '\n': - buf[pos] = '\\' buf[pos+1] = 'n' + buf[pos] = '\\' pos += 2 case '\r': - buf[pos] = '\\' buf[pos+1] = 'r' + buf[pos] = '\\' pos += 2 case '\x1a': - buf[pos] = '\\' buf[pos+1] = 'Z' + buf[pos] = '\\' pos += 2 case '\'': - buf[pos] = '\\' buf[pos+1] = '\'' + buf[pos] = '\\' pos += 2 case '"': - buf[pos] = '\\' buf[pos+1] = '"' + buf[pos] = '\\' pos += 2 case '\\': - buf[pos] = '\\' buf[pos+1] = '\\' + buf[pos] = '\\' pos += 2 default: buf[pos] = c @@ -744,8 +729,8 @@ func escapeBytesQuotes(buf, v []byte) []byte { for _, c := range v { if c == '\'' { - buf[pos] = '\'' buf[pos+1] = '\'' + buf[pos] = '\'' pos += 2 } else { buf[pos] = c @@ -764,8 +749,8 @@ func escapeStringQuotes(buf []byte, v string) []byte { for i := 0; i < len(v); i++ { c := v[i] if c == '\'' { - buf[pos] = '\'' buf[pos+1] = '\'' + buf[pos] = '\'' pos += 2 } else { buf[pos] = c @@ -796,39 +781,10 @@ func (*noCopy) Lock() {} // https://github.com/golang/go/issues/26165 func (*noCopy) Unlock() {} -// atomicBool is a wrapper around uint32 for usage as a boolean value with -// atomic access. -type atomicBool struct { - _noCopy noCopy - value uint32 -} - -// IsSet returns whether the current boolean value is true -func (ab *atomicBool) IsSet() bool { - return atomic.LoadUint32(&ab.value) > 0 -} - -// Set sets the value of the bool regardless of the previous value -func (ab *atomicBool) Set(value bool) { - if value { - atomic.StoreUint32(&ab.value, 1) - } else { - atomic.StoreUint32(&ab.value, 0) - } -} - -// TrySet sets the value of the bool and returns whether the value changed -func (ab *atomicBool) TrySet(value bool) bool { - if value { - return atomic.SwapUint32(&ab.value, 1) == 0 - } - return atomic.SwapUint32(&ab.value, 0) > 0 -} - // atomicError is a wrapper for atomically accessed error values type atomicError struct { - _noCopy noCopy - value atomic.Value + _ noCopy + value atomic.Value } // Set sets the error value regardless of the previous value. diff --git a/utils_test.go b/utils_test.go index b0069251e..42a88393c 100644 --- a/utils_test.go +++ b/utils_test.go @@ -173,66 +173,6 @@ func TestEscapeQuotes(t *testing.T) { expect("foo\"bar", "foo\"bar") // not affected } -func TestAtomicBool(t *testing.T) { - var ab atomicBool - if ab.IsSet() { - t.Fatal("Expected value to be false") - } - - ab.Set(true) - if ab.value != 1 { - t.Fatal("Set(true) did not set value to 1") - } - if !ab.IsSet() { - t.Fatal("Expected value to be true") - } - - ab.Set(true) - if !ab.IsSet() { - t.Fatal("Expected value to be true") - } - - ab.Set(false) - if ab.value != 0 { - t.Fatal("Set(false) did not set value to 0") - } - if ab.IsSet() { - t.Fatal("Expected value to be false") - } - - ab.Set(false) - if ab.IsSet() { - t.Fatal("Expected value to be false") - } - if ab.TrySet(false) { - t.Fatal("Expected TrySet(false) to fail") - } - if !ab.TrySet(true) { - t.Fatal("Expected TrySet(true) to succeed") - } - if !ab.IsSet() { - t.Fatal("Expected value to be true") - } - - ab.Set(true) - if !ab.IsSet() { - t.Fatal("Expected value to be true") - } - if ab.TrySet(true) { - t.Fatal("Expected TrySet(true) to fail") - } - if !ab.TrySet(false) { - t.Fatal("Expected TrySet(false) to succeed") - } - if ab.IsSet() { - t.Fatal("Expected value to be false") - } - - // we've "tested" them ¯\_(ツ)_/¯ - ab._noCopy.Lock() - defer ab._noCopy.Unlock() -} - func TestAtomicError(t *testing.T) { var ae atomicError if ae.Value() != nil { @@ -297,8 +237,10 @@ func TestIsolationLevelMapping(t *testing.T) { func TestAppendDateTime(t *testing.T) { tests := []struct { - t time.Time - str string + t time.Time + str string + timeTruncate time.Duration + expectedErr bool }{ { t: time.Date(1234, 5, 6, 0, 0, 0, 0, time.UTC), @@ -336,34 +278,75 @@ func TestAppendDateTime(t *testing.T) { t: time.Date(1, 1, 1, 0, 0, 0, 0, time.UTC), str: "0001-01-01", }, + // Truncated time + { + t: time.Date(1234, 5, 6, 0, 0, 0, 0, time.UTC), + str: "1234-05-06", + timeTruncate: time.Second, + }, + { + t: time.Date(4567, 12, 31, 12, 0, 0, 0, time.UTC), + str: "4567-12-31 12:00:00", + timeTruncate: time.Minute, + }, + { + t: time.Date(2020, 5, 30, 12, 34, 0, 0, time.UTC), + str: "2020-05-30 12:34:00", + timeTruncate: 0, + }, + { + t: time.Date(2020, 5, 30, 12, 34, 56, 0, time.UTC), + str: "2020-05-30 12:34:56", + timeTruncate: time.Second, + }, + { + t: time.Date(2020, 5, 30, 22, 33, 44, 123000000, time.UTC), + str: "2020-05-30 22:33:44", + timeTruncate: time.Second, + }, + { + t: time.Date(2020, 5, 30, 22, 33, 44, 123456000, time.UTC), + str: "2020-05-30 22:33:44.123", + timeTruncate: time.Millisecond, + }, + { + t: time.Date(2020, 5, 30, 22, 33, 44, 123456789, time.UTC), + str: "2020-05-30 22:33:44", + timeTruncate: time.Second, + }, + { + t: time.Date(9999, 12, 31, 23, 59, 59, 999999999, time.UTC), + str: "9999-12-31 23:59:59.999999999", + timeTruncate: 0, + }, + { + t: time.Date(1, 1, 1, 1, 1, 1, 1, time.UTC), + str: "0001-01-01", + timeTruncate: 365 * 24 * time.Hour, + }, + // year out of range + { + t: time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC), + expectedErr: true, + }, + { + t: time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC), + expectedErr: true, + }, } for _, v := range tests { buf := make([]byte, 0, 32) - buf, _ = appendDateTime(buf, v.t) + buf, err := appendDateTime(buf, v.t, v.timeTruncate) + if err != nil { + if !v.expectedErr { + t.Errorf("appendDateTime(%v) returned an error: %v", v.t, err) + } + continue + } if str := string(buf); str != v.str { t.Errorf("appendDateTime(%v), have: %s, want: %s", v.t, str, v.str) } } - - // year out of range - { - v := time.Date(0, 1, 1, 0, 0, 0, 0, time.UTC) - buf := make([]byte, 0, 32) - _, err := appendDateTime(buf, v) - if err == nil { - t.Error("want an error") - return - } - } - { - v := time.Date(10000, 1, 1, 0, 0, 0, 0, time.UTC) - buf := make([]byte, 0, 32) - _, err := appendDateTime(buf, v) - if err == nil { - t.Error("want an error") - return - } - } } func TestParseDateTime(t *testing.T) { @@ -440,6 +423,33 @@ func TestParseDateTime(t *testing.T) { } } +func TestInvalidDateTime(t *testing.T) { + cases := []struct { + name string + str string + want time.Time + }{ + { + name: "parse datetime without day", + str: "0000-00-00 21:30:45", + want: time.Date(0, 0, 0, 21, 30, 45, 0, time.UTC), + }, + } + + for _, cc := range cases { + t.Run(cc.name, func(t *testing.T) { + got, err := parseDateTime([]byte(cc.str), time.UTC) + if err != nil { + t.Fatal(err) + } + + if !cc.want.Equal(got) { + t.Fatalf("want: %v, but got %v", cc.want, got) + } + }) + } +} + func TestParseDateTimeFail(t *testing.T) { cases := []struct { name string