diff --git a/vendor/github.com/boltdb/bolt/.gitrepo b/vendor/github.com/boltdb/bolt/.gitrepo index c482d5515..cc57812be 100644 --- a/vendor/github.com/boltdb/bolt/.gitrepo +++ b/vendor/github.com/boltdb/bolt/.gitrepo @@ -4,8 +4,8 @@ ; git-subrepo command. See https://github.com/git-commands/git-subrepo#readme ; [subrepo] - remote = http://github.com/boltdb/bolt.git + remote = https://github.com/boltdb/bolt.git branch = master - commit = 05e441d7b3ded9164c5b912521504e7711dd0ba2 - parent = 99c9d17eb5416a4fdce216714d4585bb0847a582 + commit = e9cf4fae01b5a8ff89d0ec6b32f0d9c9f79aefdd + parent = bc2ebfabb1c756fb69dbcf23f4806b845adb3d1f cmdver = 0.3.0 diff --git a/vendor/github.com/boltdb/bolt/README.md b/vendor/github.com/boltdb/bolt/README.md index 36967f431..d7f80e977 100644 --- a/vendor/github.com/boltdb/bolt/README.md +++ b/vendor/github.com/boltdb/bolt/README.md @@ -15,11 +15,11 @@ and setting values. That's it. ## Project Status -Bolt is stable and the API is fixed. Full unit test coverage and randomized -black box testing are used to ensure database consistency and thread safety. -Bolt is currently in high-load production environments serving databases as -large as 1TB. Many companies such as Shopify and Heroku use Bolt-backed -services every day. +Bolt is stable, the API is fixed, and the file format is fixed. Full unit +test coverage and randomized black box testing are used to ensure database +consistency and thread safety. Bolt is currently used in high-load production +environments serving databases as large as 1TB. Many companies such as +Shopify and Heroku use Bolt-backed services every day. ## Table of Contents @@ -209,7 +209,7 @@ and then safely close your transaction if an error is returned. This is the recommended way to use Bolt transactions. However, sometimes you may want to manually start and end your transactions. -You can use the `Tx.Begin()` function directly but **please** be sure to close +You can use the `DB.Begin()` function directly but **please** be sure to close the transaction. ```go @@ -313,7 +313,7 @@ func (s *Store) CreateUser(u *User) error { // Generate ID for the user. // This returns an error only if the Tx is closed or not writeable. // That can't happen in an Update() call so I ignore the error check. - id, _ = b.NextSequence() + id, _ := b.NextSequence() u.ID = int(id) // Marshal user data into bytes. @@ -395,7 +395,7 @@ db.View(func(tx *bolt.Tx) error { c := tx.Bucket([]byte("MyBucket")).Cursor() prefix := []byte("1234") - for k, v := c.Seek(prefix); bytes.HasPrefix(k, prefix); k, v = c.Next() { + for k, v := c.Seek(prefix); k != nil && bytes.HasPrefix(k, prefix); k, v = c.Next() { fmt.Printf("key=%s, value=%s\n", k, v) } @@ -448,6 +448,10 @@ db.View(func(tx *bolt.Tx) error { }) ``` +Please note that keys and values in `ForEach()` are only valid while +the transaction is open. If you need to use a key or value outside of +the transaction, you must use `copy()` to copy it to another byte +slice. ### Nested buckets @@ -460,6 +464,55 @@ func (*Bucket) CreateBucketIfNotExists(key []byte) (*Bucket, error) func (*Bucket) DeleteBucket(key []byte) error ``` +Say you had a multi-tenant application where the root level bucket was the account bucket. Inside of this bucket was a sequence of accounts which themselves are buckets. And inside the sequence bucket you could have many buckets pertaining to the Account itself (Users, Notes, etc) isolating the information into logical groupings. + +```go + +// createUser creates a new user in the given account. +func createUser(accountID int, u *User) error { + // Start the transaction. + tx, err := db.Begin(true) + if err != nil { + return err + } + defer tx.Rollback() + + // Retrieve the root bucket for the account. + // Assume this has already been created when the account was set up. + root := tx.Bucket([]byte(strconv.FormatUint(accountID, 10))) + + // Setup the users bucket. + bkt, err := root.CreateBucketIfNotExists([]byte("USERS")) + if err != nil { + return err + } + + // Generate an ID for the new user. + userID, err := bkt.NextSequence() + if err != nil { + return err + } + u.ID = userID + + // Marshal and save the encoded user. + if buf, err := json.Marshal(u); err != nil { + return err + } else if err := bkt.Put([]byte(strconv.FormatUint(u.ID, 10)), buf); err != nil { + return err + } + + // Commit the transaction. + if err := tx.Commit(); err != nil { + return err + } + + return nil +} + +``` + + + ### Database backups @@ -715,6 +768,9 @@ Here are a few things to note when evaluating and using Bolt: can be reused by a new page or can be unmapped from virtual memory and you'll see an `unexpected fault address` panic when accessing it. +* Bolt uses an exclusive write lock on the database file so it cannot be + shared by multiple processes. + * Be careful when using `Bucket.FillPercent`. Setting a high fill percent for buckets that have random inserts will cause your database to have very poor page utilization. @@ -807,6 +863,7 @@ them via pull request. Below is a list of public, open source projects that use Bolt: +* [BoltDbWeb](https://github.com/evnix/boltdbweb) - A web based GUI for BoltDB files. * [Operation Go: A Routine Mission](http://gocode.io) - An online programming game for Golang using Bolt for user accounts and a leaderboard. * [Bazil](https://bazil.org/) - A file system that lets your data reside where it is most convenient for it to reside. * [DVID](https://github.com/janelia-flyem/dvid) - Added Bolt as optional storage engine and testing it against Basho-tuned leveldb. @@ -825,7 +882,6 @@ Below is a list of public, open source projects that use Bolt: * [cayley](https://github.com/google/cayley) - Cayley is an open-source graph database using Bolt as optional backend. * [bleve](http://www.blevesearch.com/) - A pure Go search engine similar to ElasticSearch that uses Bolt as the default storage backend. * [tentacool](https://github.com/optiflows/tentacool) - REST api server to manage system stuff (IP, DNS, Gateway...) on a linux server. -* [SkyDB](https://github.com/skydb/sky) - Behavioral analytics database. * [Seaweed File System](https://github.com/chrislusf/seaweedfs) - Highly scalable distributed key~file system with O(1) disk read. * [InfluxDB](https://influxdata.com) - Scalable datastore for metrics, events, and real-time analytics. * [Freehold](http://tshannon.bitbucket.org/freehold/) - An open, secure, and lightweight platform for your files and data. @@ -842,10 +898,18 @@ Below is a list of public, open source projects that use Bolt: * [Go Report Card](https://goreportcard.com/) - Go code quality report cards as a (free and open source) service. * [Boltdb Boilerplate](https://github.com/bobintornado/boltdb-boilerplate) - Boilerplate wrapper around bolt aiming to make simple calls one-liners. * [lru](https://github.com/crowdriff/lru) - Easy to use Bolt-backed Least-Recently-Used (LRU) read-through cache with chainable remote stores. -* [Storm](https://github.com/asdine/storm) - A simple ORM around BoltDB. +* [Storm](https://github.com/asdine/storm) - Simple and powerful ORM for BoltDB. * [GoWebApp](https://github.com/josephspurrier/gowebapp) - A basic MVC web application in Go using BoltDB. * [SimpleBolt](https://github.com/xyproto/simplebolt) - A simple way to use BoltDB. Deals mainly with strings. * [Algernon](https://github.com/xyproto/algernon) - A HTTP/2 web server with built-in support for Lua. Uses BoltDB as the default database backend. * [MuLiFS](https://github.com/dankomiocevic/mulifs) - Music Library Filesystem creates a filesystem to organise your music files. +* [GoShort](https://github.com/pankajkhairnar/goShort) - GoShort is a URL shortener written in Golang and BoltDB for persistent key/value storage and for routing it's using high performent HTTPRouter. +* [torrent](https://github.com/anacrolix/torrent) - Full-featured BitTorrent client package and utilities in Go. BoltDB is a storage backend in development. +* [gopherpit](https://github.com/gopherpit/gopherpit) - A web service to manage Go remote import paths with custom domains +* [bolter](https://github.com/hasit/bolter) - Command-line app for viewing BoltDB file in your terminal. +* [btcwallet](https://github.com/btcsuite/btcwallet) - A bitcoin wallet. +* [dcrwallet](https://github.com/decred/dcrwallet) - A wallet for the Decred cryptocurrency. +* [Ironsmith](https://github.com/timshannon/ironsmith) - A simple, script-driven continuous integration (build - > test -> release) tool, with no external dependencies +* [BoltHold](https://github.com/timshannon/bolthold) - An embeddable NoSQL store for Go types built on BoltDB If you are using Bolt in a project please send a pull request to add it to the list. diff --git a/vendor/github.com/boltdb/bolt/bolt_386.go b/vendor/github.com/boltdb/bolt/bolt_386.go index e659bfb91..820d533c1 100644 --- a/vendor/github.com/boltdb/bolt/bolt_386.go +++ b/vendor/github.com/boltdb/bolt/bolt_386.go @@ -5,3 +5,6 @@ const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_amd64.go b/vendor/github.com/boltdb/bolt/bolt_amd64.go index cca6b7eb7..98fafdb47 100644 --- a/vendor/github.com/boltdb/bolt/bolt_amd64.go +++ b/vendor/github.com/boltdb/bolt/bolt_amd64.go @@ -5,3 +5,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_arm.go b/vendor/github.com/boltdb/bolt/bolt_arm.go index e659bfb91..7e5cb4b94 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm.go +++ b/vendor/github.com/boltdb/bolt/bolt_arm.go @@ -1,7 +1,28 @@ package bolt +import "unsafe" + // maxMapSize represents the largest mmap size supported by Bolt. const maxMapSize = 0x7FFFFFFF // 2GB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0xFFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned bool + +func init() { + // Simple check to see whether this arch handles unaligned load/stores + // correctly. + + // ARM9 and older devices require load/stores to be from/to aligned + // addresses. If not, the lower 2 bits are cleared and that address is + // read in a jumbled up order. + + // See http://infocenter.arm.com/help/index.jsp?topic=/com.arm.doc.faqs/ka15414.html + + raw := [6]byte{0xfe, 0xef, 0x11, 0x22, 0x22, 0x11} + val := *(*uint32)(unsafe.Pointer(uintptr(unsafe.Pointer(&raw)) + 2)) + + brokenUnaligned = val != 0x11222211 +} diff --git a/vendor/github.com/boltdb/bolt/bolt_arm64.go b/vendor/github.com/boltdb/bolt/bolt_arm64.go index 6d2309352..b26d84f91 100644 --- a/vendor/github.com/boltdb/bolt/bolt_arm64.go +++ b/vendor/github.com/boltdb/bolt/bolt_arm64.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64.go b/vendor/github.com/boltdb/bolt/bolt_ppc64.go index 2dc6be02e..9331d9771 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64.go +++ b/vendor/github.com/boltdb/bolt/bolt_ppc64.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go b/vendor/github.com/boltdb/bolt/bolt_ppc64le.go index 8351e129f..8c143bc5d 100644 --- a/vendor/github.com/boltdb/bolt/bolt_ppc64le.go +++ b/vendor/github.com/boltdb/bolt/bolt_ppc64le.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_s390x.go b/vendor/github.com/boltdb/bolt/bolt_s390x.go index f4dd26bbb..d7c39af92 100644 --- a/vendor/github.com/boltdb/bolt/bolt_s390x.go +++ b/vendor/github.com/boltdb/bolt/bolt_s390x.go @@ -7,3 +7,6 @@ const maxMapSize = 0xFFFFFFFFFFFF // 256TB // maxAllocSize is the size used when creating array pointers. const maxAllocSize = 0x7FFFFFFF + +// Are unaligned load/stores broken on this arch? +var brokenUnaligned = false diff --git a/vendor/github.com/boltdb/bolt/bolt_windows.go b/vendor/github.com/boltdb/bolt/bolt_windows.go index d538e6afd..b00fb0720 100644 --- a/vendor/github.com/boltdb/bolt/bolt_windows.go +++ b/vendor/github.com/boltdb/bolt/bolt_windows.go @@ -89,7 +89,7 @@ func flock(db *DB, mode os.FileMode, exclusive bool, timeout time.Duration) erro func funlock(db *DB) error { err := unlockFileEx(syscall.Handle(db.lockfile.Fd()), 0, 1, 0, &syscall.Overlapped{}) db.lockfile.Close() - os.Remove(db.path+lockExt) + os.Remove(db.path + lockExt) return err } diff --git a/vendor/github.com/boltdb/bolt/bucket.go b/vendor/github.com/boltdb/bolt/bucket.go index d2f8c524e..0c5bf2746 100644 --- a/vendor/github.com/boltdb/bolt/bucket.go +++ b/vendor/github.com/boltdb/bolt/bucket.go @@ -130,9 +130,17 @@ func (b *Bucket) Bucket(name []byte) *Bucket { func (b *Bucket) openBucket(value []byte) *Bucket { var child = newBucket(b.tx) + // If unaligned load/stores are broken on this arch and value is + // unaligned simply clone to an aligned byte array. + unaligned := brokenUnaligned && uintptr(unsafe.Pointer(&value[0]))&3 != 0 + + if unaligned { + value = cloneBytes(value) + } + // If this is a writable transaction then we need to copy the bucket entry. // Read-only transactions can point directly at the mmap entry. - if b.tx.writable { + if b.tx.writable && !unaligned { child.bucket = &bucket{} *child.bucket = *(*bucket)(unsafe.Pointer(&value[0])) } else { @@ -167,9 +175,8 @@ func (b *Bucket) CreateBucket(key []byte) (*Bucket, error) { if bytes.Equal(key, k) { if (flags & bucketLeafFlag) != 0 { return nil, ErrBucketExists - } else { - return nil, ErrIncompatibleValue } + return nil, ErrIncompatibleValue } // Create empty, inline bucket. @@ -329,6 +336,28 @@ func (b *Bucket) Delete(key []byte) error { return nil } +// Sequence returns the current integer for the bucket without incrementing it. +func (b *Bucket) Sequence() uint64 { return b.bucket.sequence } + +// SetSequence updates the sequence number for the bucket. +func (b *Bucket) SetSequence(v uint64) error { + if b.tx.db == nil { + return ErrTxClosed + } else if !b.Writable() { + return ErrTxNotWritable + } + + // Materialize the root node if it hasn't been already so that the + // bucket will be saved during commit. + if b.rootNode == nil { + _ = b.node(b.root, nil) + } + + // Increment and return the sequence. + b.bucket.sequence = v + return nil +} + // NextSequence returns an autoincrementing integer for the bucket. func (b *Bucket) NextSequence() (uint64, error) { if b.tx.db == nil { diff --git a/vendor/github.com/boltdb/bolt/bucket_test.go b/vendor/github.com/boltdb/bolt/bucket_test.go index 528fec24d..cddbe2713 100644 --- a/vendor/github.com/boltdb/bolt/bucket_test.go +++ b/vendor/github.com/boltdb/bolt/bucket_test.go @@ -782,6 +782,48 @@ func TestBucket_DeleteBucket_IncompatibleValue(t *testing.T) { } } +// Ensure bucket can set and update its sequence number. +func TestBucket_Sequence(t *testing.T) { + db := MustOpenDB() + defer db.MustClose() + + if err := db.Update(func(tx *bolt.Tx) error { + bkt, err := tx.CreateBucket([]byte("0")) + if err != nil { + t.Fatal(err) + } + + // Retrieve sequence. + if v := bkt.Sequence(); v != 0 { + t.Fatalf("unexpected sequence: %d", v) + } + + // Update sequence. + if err := bkt.SetSequence(1000); err != nil { + t.Fatal(err) + } + + // Read sequence again. + if v := bkt.Sequence(); v != 1000 { + t.Fatalf("unexpected sequence: %d", v) + } + + return nil + }); err != nil { + t.Fatal(err) + } + + // Verify sequence in separate transaction. + if err := db.View(func(tx *bolt.Tx) error { + if v := tx.Bucket([]byte("0")).Sequence(); v != 1000 { + t.Fatalf("unexpected sequence: %d", v) + } + return nil + }); err != nil { + t.Fatal(err) + } +} + // Ensure that a bucket can return an autoincrementing sequence. func TestBucket_NextSequence(t *testing.T) { db := MustOpenDB() diff --git a/vendor/github.com/boltdb/bolt/cmd/bolt/main.go b/vendor/github.com/boltdb/bolt/cmd/bolt/main.go index b96e6f735..057eca50a 100644 --- a/vendor/github.com/boltdb/bolt/cmd/bolt/main.go +++ b/vendor/github.com/boltdb/bolt/cmd/bolt/main.go @@ -102,6 +102,8 @@ func (m *Main) Run(args ...string) error { return newBenchCommand(m).Run(args[1:]...) case "check": return newCheckCommand(m).Run(args[1:]...) + case "compact": + return newCompactCommand(m).Run(args[1:]...) case "dump": return newDumpCommand(m).Run(args[1:]...) case "info": @@ -130,6 +132,7 @@ The commands are: bench run synthetic benchmark against bolt check verifies integrity of bolt database + compact copies a bolt database, compacting it in the process info print basic info help print this screen pages print list of pages with their types @@ -356,7 +359,7 @@ func (cmd *DumpCommand) Run(args ...string) error { return nil } -// PrintPage prints a given page as hexidecimal. +// PrintPage prints a given page as hexadecimal. func (cmd *DumpCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { const bytesPerLineN = 16 @@ -406,7 +409,7 @@ func (cmd *DumpCommand) Usage() string { return strings.TrimLeft(` usage: bolt dump -page PAGEID PATH -Dump prints a hexidecimal dump of a single page. +Dump prints a hexadecimal dump of a single page. `, "\n") } @@ -539,9 +542,9 @@ func (cmd *PageCommand) PrintLeaf(w io.Writer, buf []byte) error { b := (*bucket)(unsafe.Pointer(&e.value()[0])) v = fmt.Sprintf("", b.root, b.sequence) } else if isPrintable(string(e.value())) { - k = fmt.Sprintf("%q", string(e.value())) + v = fmt.Sprintf("%q", string(e.value())) } else { - k = fmt.Sprintf("%x", string(e.value())) + v = fmt.Sprintf("%x", string(e.value())) } fmt.Fprintf(w, "%s: %s\n", k, v) @@ -593,7 +596,7 @@ func (cmd *PageCommand) PrintFreelist(w io.Writer, buf []byte) error { return nil } -// PrintPage prints a given page as hexidecimal. +// PrintPage prints a given page as hexadecimal. func (cmd *PageCommand) PrintPage(w io.Writer, r io.ReaderAt, pageID int, pageSize int) error { const bytesPerLineN = 16 @@ -1530,3 +1533,208 @@ func (n *leafPageElement) value() []byte { buf := (*[maxAllocSize]byte)(unsafe.Pointer(n)) return buf[n.pos+n.ksize : n.pos+n.ksize+n.vsize] } + +// CompactCommand represents the "compact" command execution. +type CompactCommand struct { + Stdin io.Reader + Stdout io.Writer + Stderr io.Writer + + SrcPath string + DstPath string + TxMaxSize int64 +} + +// newCompactCommand returns a CompactCommand. +func newCompactCommand(m *Main) *CompactCommand { + return &CompactCommand{ + Stdin: m.Stdin, + Stdout: m.Stdout, + Stderr: m.Stderr, + } +} + +// Run executes the command. +func (cmd *CompactCommand) Run(args ...string) (err error) { + // Parse flags. + fs := flag.NewFlagSet("", flag.ContinueOnError) + fs.SetOutput(ioutil.Discard) + fs.StringVar(&cmd.DstPath, "o", "", "") + fs.Int64Var(&cmd.TxMaxSize, "tx-max-size", 65536, "") + if err := fs.Parse(args); err == flag.ErrHelp { + fmt.Fprintln(cmd.Stderr, cmd.Usage()) + return ErrUsage + } else if err != nil { + return err + } else if cmd.DstPath == "" { + return fmt.Errorf("output file required") + } + + // Require database paths. + cmd.SrcPath = fs.Arg(0) + if cmd.SrcPath == "" { + return ErrPathRequired + } + + // Ensure source file exists. + fi, err := os.Stat(cmd.SrcPath) + if os.IsNotExist(err) { + return ErrFileNotFound + } else if err != nil { + return err + } + initialSize := fi.Size() + + // Open source database. + src, err := bolt.Open(cmd.SrcPath, 0444, nil) + if err != nil { + return err + } + defer src.Close() + + // Open destination database. + dst, err := bolt.Open(cmd.DstPath, fi.Mode(), nil) + if err != nil { + return err + } + defer dst.Close() + + // Run compaction. + if err := cmd.compact(dst, src); err != nil { + return err + } + + // Report stats on new size. + fi, err = os.Stat(cmd.DstPath) + if err != nil { + return err + } else if fi.Size() == 0 { + return fmt.Errorf("zero db size") + } + fmt.Fprintf(cmd.Stdout, "%d -> %d bytes (gain=%.2fx)\n", initialSize, fi.Size(), float64(initialSize)/float64(fi.Size())) + + return nil +} + +func (cmd *CompactCommand) compact(dst, src *bolt.DB) error { + // commit regularly, or we'll run out of memory for large datasets if using one transaction. + var size int64 + tx, err := dst.Begin(true) + if err != nil { + return err + } + defer tx.Rollback() + + if err := cmd.walk(src, func(keys [][]byte, k, v []byte, seq uint64) error { + // On each key/value, check if we have exceeded tx size. + sz := int64(len(k) + len(v)) + if size+sz > cmd.TxMaxSize && cmd.TxMaxSize != 0 { + // Commit previous transaction. + if err := tx.Commit(); err != nil { + return err + } + + // Start new transaction. + tx, err = dst.Begin(true) + if err != nil { + return err + } + size = 0 + } + size += sz + + // Create bucket on the root transaction if this is the first level. + nk := len(keys) + if nk == 0 { + bkt, err := tx.CreateBucket(k) + if err != nil { + return err + } + if err := bkt.SetSequence(seq); err != nil { + return err + } + return nil + } + + // Create buckets on subsequent levels, if necessary. + b := tx.Bucket(keys[0]) + if nk > 1 { + for _, k := range keys[1:] { + b = b.Bucket(k) + } + } + + // If there is no value then this is a bucket call. + if v == nil { + bkt, err := b.CreateBucket(k) + if err != nil { + return err + } + if err := bkt.SetSequence(seq); err != nil { + return err + } + return nil + } + + // Otherwise treat it as a key/value pair. + return b.Put(k, v) + }); err != nil { + return err + } + + return tx.Commit() +} + +// walkFunc is the type of the function called for keys (buckets and "normal" +// values) discovered by Walk. keys is the list of keys to descend to the bucket +// owning the discovered key/value pair k/v. +type walkFunc func(keys [][]byte, k, v []byte, seq uint64) error + +// walk walks recursively the bolt database db, calling walkFn for each key it finds. +func (cmd *CompactCommand) walk(db *bolt.DB, walkFn walkFunc) error { + return db.View(func(tx *bolt.Tx) error { + return tx.ForEach(func(name []byte, b *bolt.Bucket) error { + return cmd.walkBucket(b, nil, name, nil, b.Sequence(), walkFn) + }) + }) +} + +func (cmd *CompactCommand) walkBucket(b *bolt.Bucket, keypath [][]byte, k, v []byte, seq uint64, fn walkFunc) error { + // Execute callback. + if err := fn(keypath, k, v, seq); err != nil { + return err + } + + // If this is not a bucket then stop. + if v != nil { + return nil + } + + // Iterate over each child key/value. + keypath = append(keypath, k) + return b.ForEach(func(k, v []byte) error { + if v == nil { + bkt := b.Bucket(k) + return cmd.walkBucket(bkt, keypath, k, nil, bkt.Sequence(), fn) + } + return cmd.walkBucket(b, keypath, k, v, b.Sequence(), fn) + }) +} + +// Usage returns the help message. +func (cmd *CompactCommand) Usage() string { + return strings.TrimLeft(` +usage: bolt compact [options] -o DST SRC + +Compact opens a database at SRC path and walks it recursively, copying keys +as they are found from all buckets, to a newly created database at DST path. + +The original database is left untouched. + +Additional options include: + + -tx-max-size NUM + Specifies the maximum size of individual transactions. + Defaults to 64KB. +`, "\n") +} diff --git a/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go b/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go index c378b790d..0a11ff339 100644 --- a/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go +++ b/vendor/github.com/boltdb/bolt/cmd/bolt/main_test.go @@ -2,7 +2,12 @@ package main_test import ( "bytes" + crypto "crypto/rand" + "encoding/binary" + "fmt" + "io" "io/ioutil" + "math/rand" "os" "strconv" "testing" @@ -183,3 +188,169 @@ func (db *DB) Close() error { defer os.Remove(db.Path) return db.DB.Close() } + +func TestCompactCommand_Run(t *testing.T) { + var s int64 + if err := binary.Read(crypto.Reader, binary.BigEndian, &s); err != nil { + t.Fatal(err) + } + rand.Seed(s) + + dstdb := MustOpen(0666, nil) + dstdb.Close() + + // fill the db + db := MustOpen(0666, nil) + if err := db.Update(func(tx *bolt.Tx) error { + n := 2 + rand.Intn(5) + for i := 0; i < n; i++ { + k := []byte(fmt.Sprintf("b%d", i)) + b, err := tx.CreateBucketIfNotExists(k) + if err != nil { + return err + } + if err := b.SetSequence(uint64(i)); err != nil { + return err + } + if err := fillBucket(b, append(k, '.')); err != nil { + return err + } + } + return nil + }); err != nil { + db.Close() + t.Fatal(err) + } + + // make the db grow by adding large values, and delete them. + if err := db.Update(func(tx *bolt.Tx) error { + b, err := tx.CreateBucketIfNotExists([]byte("large_vals")) + if err != nil { + return err + } + n := 5 + rand.Intn(5) + for i := 0; i < n; i++ { + v := make([]byte, 1000*1000*(1+rand.Intn(5))) + _, err := crypto.Read(v) + if err != nil { + return err + } + if err := b.Put([]byte(fmt.Sprintf("l%d", i)), v); err != nil { + return err + } + } + return nil + }); err != nil { + db.Close() + t.Fatal(err) + } + if err := db.Update(func(tx *bolt.Tx) error { + c := tx.Bucket([]byte("large_vals")).Cursor() + for k, _ := c.First(); k != nil; k, _ = c.Next() { + if err := c.Delete(); err != nil { + return err + } + } + return tx.DeleteBucket([]byte("large_vals")) + }); err != nil { + db.Close() + t.Fatal(err) + } + db.DB.Close() + defer db.Close() + defer dstdb.Close() + + dbChk, err := chkdb(db.Path) + if err != nil { + t.Fatal(err) + } + + m := NewMain() + if err := m.Run("compact", "-o", dstdb.Path, db.Path); err != nil { + t.Fatal(err) + } + + dbChkAfterCompact, err := chkdb(db.Path) + if err != nil { + t.Fatal(err) + } + + dstdbChk, err := chkdb(dstdb.Path) + if err != nil { + t.Fatal(err) + } + + if !bytes.Equal(dbChk, dbChkAfterCompact) { + t.Error("the original db has been touched") + } + if !bytes.Equal(dbChk, dstdbChk) { + t.Error("the compacted db data isn't the same than the original db") + } +} + +func fillBucket(b *bolt.Bucket, prefix []byte) error { + n := 10 + rand.Intn(50) + for i := 0; i < n; i++ { + v := make([]byte, 10*(1+rand.Intn(4))) + _, err := crypto.Read(v) + if err != nil { + return err + } + k := append(prefix, []byte(fmt.Sprintf("k%d", i))...) + if err := b.Put(k, v); err != nil { + return err + } + } + // limit depth of subbuckets + s := 2 + rand.Intn(4) + if len(prefix) > (2*s + 1) { + return nil + } + n = 1 + rand.Intn(3) + for i := 0; i < n; i++ { + k := append(prefix, []byte(fmt.Sprintf("b%d", i))...) + sb, err := b.CreateBucket(k) + if err != nil { + return err + } + if err := fillBucket(sb, append(k, '.')); err != nil { + return err + } + } + return nil +} + +func chkdb(path string) ([]byte, error) { + db, err := bolt.Open(path, 0666, nil) + if err != nil { + return nil, err + } + defer db.Close() + var buf bytes.Buffer + err = db.View(func(tx *bolt.Tx) error { + return tx.ForEach(func(name []byte, b *bolt.Bucket) error { + return walkBucket(b, name, nil, &buf) + }) + }) + if err != nil { + return nil, err + } + return buf.Bytes(), nil +} + +func walkBucket(parent *bolt.Bucket, k []byte, v []byte, w io.Writer) error { + if _, err := fmt.Fprintf(w, "%d:%x=%x\n", parent.Sequence(), k, v); err != nil { + return err + } + + // not a bucket, exit. + if v != nil { + return nil + } + return parent.ForEach(func(k, v []byte) error { + if v == nil { + return walkBucket(parent.Bucket(k), k, nil, w) + } + return walkBucket(parent, k, v, w) + }) +} diff --git a/vendor/github.com/boltdb/bolt/db.go b/vendor/github.com/boltdb/bolt/db.go index 1223493ca..f352ff14f 100644 --- a/vendor/github.com/boltdb/bolt/db.go +++ b/vendor/github.com/boltdb/bolt/db.go @@ -552,7 +552,10 @@ func (db *DB) removeTx(tx *Tx) { // Remove the transaction. for i, t := range db.txs { if t == tx { - db.txs = append(db.txs[:i], db.txs[i+1:]...) + last := len(db.txs) - 1 + db.txs[i] = db.txs[last] + db.txs[last] = nil + db.txs = db.txs[:last] break } } @@ -952,7 +955,7 @@ func (s *Stats) Sub(other *Stats) Stats { diff.PendingPageN = s.PendingPageN diff.FreeAlloc = s.FreeAlloc diff.FreelistInuse = s.FreelistInuse - diff.TxN = other.TxN - s.TxN + diff.TxN = s.TxN - other.TxN diff.TxStats = s.TxStats.Sub(&other.TxStats) return diff } diff --git a/vendor/github.com/boltdb/bolt/db_test.go b/vendor/github.com/boltdb/bolt/db_test.go index 74ff93a9f..3034d4f47 100644 --- a/vendor/github.com/boltdb/bolt/db_test.go +++ b/vendor/github.com/boltdb/bolt/db_test.go @@ -12,7 +12,6 @@ import ( "os" "path/filepath" "regexp" - "runtime" "sort" "strings" "sync" @@ -180,69 +179,6 @@ func TestOpen_ErrChecksum(t *testing.T) { } } -// Ensure that opening an already open database file will timeout. -func TestOpen_Timeout(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("solaris fcntl locks don't support intra-process locking") - } - - path := tempfile() - - // Open a data file. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } else if db0 == nil { - t.Fatal("expected database") - } - - // Attempt to open the database again. - start := time.Now() - db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 100 * time.Millisecond}) - if err != bolt.ErrTimeout { - t.Fatalf("unexpected timeout: %s", err) - } else if db1 != nil { - t.Fatal("unexpected database") - } else if time.Since(start) <= 100*time.Millisecond { - t.Fatal("expected to wait at least timeout duration") - } - - if err := db0.Close(); err != nil { - t.Fatal(err) - } -} - -// Ensure that opening an already open database file will wait until its closed. -func TestOpen_Wait(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("solaris fcntl locks don't support intra-process locking") - } - - path := tempfile() - - // Open a data file. - db0, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } - - // Close it in just a bit. - time.AfterFunc(100*time.Millisecond, func() { _ = db0.Close() }) - - // Attempt to open the database again. - start := time.Now() - db1, err := bolt.Open(path, 0666, &bolt.Options{Timeout: 200 * time.Millisecond}) - if err != nil { - t.Fatal(err) - } else if time.Since(start) <= 100*time.Millisecond { - t.Fatal("expected to wait at least timeout duration") - } - - if err := db1.Close(); err != nil { - t.Fatal(err) - } -} - // Ensure that opening a database does not increase its size. // https://github.com/boltdb/bolt/issues/291 func TestOpen_Size(t *testing.T) { @@ -426,103 +362,6 @@ func TestOpen_FileTooSmall(t *testing.T) { } } -// Ensure that a database can be opened in read-only mode by multiple processes -// and that a database can not be opened in read-write mode and in read-only -// mode at the same time. -func TestOpen_ReadOnly(t *testing.T) { - if runtime.GOOS == "solaris" { - t.Skip("solaris fcntl locks don't support intra-process locking") - } - - bucket, key, value := []byte(`bucket`), []byte(`key`), []byte(`value`) - - path := tempfile() - - // Open in read-write mode. - db, err := bolt.Open(path, 0666, nil) - if err != nil { - t.Fatal(err) - } else if db.IsReadOnly() { - t.Fatal("db should not be in read only mode") - } - if err := db.Update(func(tx *bolt.Tx) error { - b, err := tx.CreateBucket(bucket) - if err != nil { - return err - } - if err := b.Put(key, value); err != nil { - t.Fatal(err) - } - return nil - }); err != nil { - t.Fatal(err) - } - if err := db.Close(); err != nil { - t.Fatal(err) - } - - // Open in read-only mode. - db0, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) - if err != nil { - t.Fatal(err) - } - - // Opening in read-write mode should return an error. - if _, err = bolt.Open(path, 0666, &bolt.Options{Timeout: time.Millisecond * 100}); err == nil { - t.Fatal("expected error") - } - - // And again (in read-only mode). - db1, err := bolt.Open(path, 0666, &bolt.Options{ReadOnly: true}) - if err != nil { - t.Fatal(err) - } - - // Verify both read-only databases are accessible. - for _, db := range []*bolt.DB{db0, db1} { - // Verify is is in read only mode indeed. - if !db.IsReadOnly() { - t.Fatal("expected read only mode") - } - - // Read-only databases should not allow updates. - if err := db.Update(func(*bolt.Tx) error { - panic(`should never get here`) - }); err != bolt.ErrDatabaseReadOnly { - t.Fatalf("unexpected error: %s", err) - } - - // Read-only databases should not allow beginning writable txns. - if _, err := db.Begin(true); err != bolt.ErrDatabaseReadOnly { - t.Fatalf("unexpected error: %s", err) - } - - // Verify the data. - if err := db.View(func(tx *bolt.Tx) error { - b := tx.Bucket(bucket) - if b == nil { - return fmt.Errorf("expected bucket `%s`", string(bucket)) - } - - got := string(b.Get(key)) - expected := string(value) - if got != expected { - return fmt.Errorf("expected `%s`, got `%s`", expected, got) - } - return nil - }); err != nil { - t.Fatal(err) - } - } - - if err := db0.Close(); err != nil { - t.Fatal(err) - } - if err := db1.Close(); err != nil { - t.Fatal(err) - } -} - // TestDB_Open_InitialMmapSize tests if having InitialMmapSize large enough // to hold data from concurrent write transaction resolves the issue that // read transaction blocks the write transaction and causes deadlock. diff --git a/vendor/github.com/boltdb/bolt/freelist.go b/vendor/github.com/boltdb/bolt/freelist.go index 0161948fc..aba48f58c 100644 --- a/vendor/github.com/boltdb/bolt/freelist.go +++ b/vendor/github.com/boltdb/bolt/freelist.go @@ -24,7 +24,12 @@ func newFreelist() *freelist { // size returns the size of the page after serialization. func (f *freelist) size() int { - return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * f.count()) + n := f.count() + if n >= 0xFFFF { + // The first element will be used to store the count. See freelist.write. + n++ + } + return pageHeaderSize + (int(unsafe.Sizeof(pgid(0))) * n) } // count returns count of pages on the freelist @@ -46,16 +51,15 @@ func (f *freelist) pending_count() int { return count } -// all returns a list of all free ids and all pending ids in one sorted list. -func (f *freelist) all() []pgid { - m := make(pgids, 0) - +// copyall copies into dst a list of all free ids and all pending ids in one sorted list. +// f.count returns the minimum length required for dst. +func (f *freelist) copyall(dst []pgid) { + m := make(pgids, 0, f.pending_count()) for _, list := range f.pending { m = append(m, list...) } - sort.Sort(m) - return pgids(f.ids).merge(m) + mergepgids(dst, f.ids, m) } // allocate returns the starting page id of a contiguous list of pages of a given size. @@ -166,12 +170,16 @@ func (f *freelist) read(p *page) { } // Copy the list of page ids from the freelist. - ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] - f.ids = make([]pgid, len(ids)) - copy(f.ids, ids) + if count == 0 { + f.ids = nil + } else { + ids := ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[idx:count] + f.ids = make([]pgid, len(ids)) + copy(f.ids, ids) - // Make sure they're sorted. - sort.Sort(pgids(f.ids)) + // Make sure they're sorted. + sort.Sort(pgids(f.ids)) + } // Rebuild the page cache. f.reindex() @@ -182,20 +190,22 @@ func (f *freelist) read(p *page) { // become free. func (f *freelist) write(p *page) error { // Combine the old free pgids and pgids waiting on an open transaction. - ids := f.all() // Update the header flag. p.flags |= freelistPageFlag // The page.count can only hold up to 64k elements so if we overflow that // number then we handle it by putting the size in the first element. - if len(ids) < 0xFFFF { - p.count = uint16(len(ids)) - copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:], ids) + lenids := f.count() + if lenids == 0 { + p.count = uint16(lenids) + } else if lenids < 0xFFFF { + p.count = uint16(lenids) + f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[:]) } else { p.count = 0xFFFF - ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(len(ids)) - copy(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:], ids) + ((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[0] = pgid(lenids) + f.copyall(((*[maxAllocSize]pgid)(unsafe.Pointer(&p.ptr)))[1:]) } return nil @@ -230,7 +240,7 @@ func (f *freelist) reload(p *page) { // reindex rebuilds the free cache based on available and pending free lists. func (f *freelist) reindex() { - f.cache = make(map[pgid]bool) + f.cache = make(map[pgid]bool, len(f.ids)) for _, id := range f.ids { f.cache[id] = true } diff --git a/vendor/github.com/boltdb/bolt/node.go b/vendor/github.com/boltdb/bolt/node.go index e9d64af81..159318b22 100644 --- a/vendor/github.com/boltdb/bolt/node.go +++ b/vendor/github.com/boltdb/bolt/node.go @@ -201,6 +201,11 @@ func (n *node) write(p *page) { } p.count = uint16(len(n.inodes)) + // Stop here if there are no items to write. + if p.count == 0 { + return + } + // Loop over each item and write it to the page. b := (*[maxAllocSize]byte)(unsafe.Pointer(&p.ptr))[n.pageElementSize()*len(n.inodes):] for i, item := range n.inodes { diff --git a/vendor/github.com/boltdb/bolt/page.go b/vendor/github.com/boltdb/bolt/page.go index 4a555286a..cde403ae8 100644 --- a/vendor/github.com/boltdb/bolt/page.go +++ b/vendor/github.com/boltdb/bolt/page.go @@ -62,6 +62,9 @@ func (p *page) leafPageElement(index uint16) *leafPageElement { // leafPageElements retrieves a list of leaf nodes. func (p *page) leafPageElements() []leafPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]leafPageElement)(unsafe.Pointer(&p.ptr)))[:] } @@ -72,6 +75,9 @@ func (p *page) branchPageElement(index uint16) *branchPageElement { // branchPageElements retrieves a list of branch nodes. func (p *page) branchPageElements() []branchPageElement { + if p.count == 0 { + return nil + } return ((*[0x7FFFFFF]branchPageElement)(unsafe.Pointer(&p.ptr)))[:] } @@ -139,12 +145,33 @@ func (a pgids) merge(b pgids) pgids { // Return the opposite slice if one is nil. if len(a) == 0 { return b - } else if len(b) == 0 { + } + if len(b) == 0 { return a } + merged := make(pgids, len(a)+len(b)) + mergepgids(merged, a, b) + return merged +} + +// mergepgids copies the sorted union of a and b into dst. +// If dst is too small, it panics. +func mergepgids(dst, a, b pgids) { + if len(dst) < len(a)+len(b) { + panic(fmt.Errorf("mergepgids bad len %d < %d + %d", len(dst), len(a), len(b))) + } + // Copy in the opposite slice if one is nil. + if len(a) == 0 { + copy(dst, b) + return + } + if len(b) == 0 { + copy(dst, a) + return + } - // Create a list to hold all elements from both lists. - merged := make(pgids, 0, len(a)+len(b)) + // Merged will hold all elements from both lists. + merged := dst[:0] // Assign lead to the slice with a lower starting value, follow to the higher value. lead, follow := a, b @@ -166,7 +193,5 @@ func (a pgids) merge(b pgids) pgids { } // Append what's left in follow. - merged = append(merged, follow...) - - return merged + _ = append(merged, follow...) } diff --git a/vendor/github.com/boltdb/bolt/quick_test.go b/vendor/github.com/boltdb/bolt/quick_test.go index 4da581775..9e27792e1 100644 --- a/vendor/github.com/boltdb/bolt/quick_test.go +++ b/vendor/github.com/boltdb/bolt/quick_test.go @@ -50,9 +50,17 @@ func (t testdata) Less(i, j int) bool { return bytes.Compare(t[i].Key, t[j].Key) func (t testdata) Generate(rand *rand.Rand, size int) reflect.Value { n := rand.Intn(qmaxitems-1) + 1 items := make(testdata, n) + used := make(map[string]bool) for i := 0; i < n; i++ { item := &items[i] - item.Key = randByteSlice(rand, 1, qmaxksize) + // Ensure that keys are unique by looping until we find one that we have not already used. + for { + item.Key = randByteSlice(rand, 1, qmaxksize) + if !used[string(item.Key)] { + used[string(item.Key)] = true + break + } + } item.Value = randByteSlice(rand, 0, qmaxvsize) } return reflect.ValueOf(items) diff --git a/vendor/github.com/boltdb/bolt/tx.go b/vendor/github.com/boltdb/bolt/tx.go index 1cfb4cde8..6700308a2 100644 --- a/vendor/github.com/boltdb/bolt/tx.go +++ b/vendor/github.com/boltdb/bolt/tx.go @@ -381,7 +381,9 @@ func (tx *Tx) Check() <-chan error { func (tx *Tx) check(ch chan error) { // Check if any pages are double freed. freed := make(map[pgid]bool) - for _, id := range tx.db.freelist.all() { + all := make([]pgid, tx.db.freelist.count()) + tx.db.freelist.copyall(all) + for _, id := range all { if freed[id] { ch <- fmt.Errorf("page %d: already freed", id) }