Skip to content

Commit

Permalink
tpch: port dbgen (#8)
Browse files Browse the repository at this point in the history
* tpc-h: add run part

Signed-off-by: mahjonp <[email protected]>
  • Loading branch information
mahjonp authored and zhouqiang-cl committed Nov 27, 2019
1 parent 91f1256 commit b36483e
Show file tree
Hide file tree
Showing 31 changed files with 5,081 additions and 9 deletions.
5 changes: 5 additions & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,8 @@ GOOS := $(if $(GOOS),$(GOOS),linux)
GOARCH := $(if $(GOARCH),$(GOARCH),amd64)
GO=GO15VENDOREXPERIMENT="1" CGO_ENABLED=0 GOOS=$(GOOS) GOARCH=$(GOARCH) GO111MODULE=on go

PACKAGE_LIST := go list ./...| grep -vE "cmd"
PACKAGES := $$($(PACKAGE_LIST))
FILES := $$(find . -name "*.go" | grep -vE "vendor")
GOFILTER := grep -vE 'vendor|render.Delims|bindata_assetfs|testutil|\.pb\.go'
GOCHECKER := $(GOFILTER) | awk '{ print } END { if (NR > 0) { exit 1 } }'
Expand All @@ -12,5 +14,8 @@ fmt:
@echo "gofmt"
@gofmt -s -l -w $(FILES) 2>&1 | $(GOCHECKER)

test:
go test ./... -cover $(PACKAGES)

build:
go build -o ./bin/go-tpc cmd/go-tpc/*
12 changes: 12 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -21,3 +21,15 @@ make
./bin/go-tpc tpcc --warehouses 4 check
```

## TPC-H

```bash
# Prepare data with scale factor 1
./bin/go-tpc tpch --sf=1 prepare
# Run TPCH workloads with result checking
./bin/go-tpc tpch --sf=1 --check=true run
# Run TPCH workloads without result checking
./bin/go-tpc tpch --sf=1 run
# Cleanup
./bin/go-tpc tpch cleanup
```
2 changes: 1 addition & 1 deletion cmd/go-tpc/main.go
Original file line number Diff line number Diff line change
Expand Up @@ -83,7 +83,7 @@ func main() {
registerTpch(rootCmd)

var cancel context.CancelFunc
globalCtx, cancel = context.WithTimeout(context.Background(), totalTime)
globalCtx, cancel = context.WithCancel(context.Background())

sc := make(chan os.Signal, 1)
signal.Notify(sc,
Expand Down
7 changes: 6 additions & 1 deletion cmd/go-tpc/tpcc.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,8 @@
package main

import (
"context"

"github.com/pingcap/go-tpc/tpcc"
"github.com/spf13/cobra"
)
Expand All @@ -15,7 +17,10 @@ func executeTpcc(action string, args []string) {
tpccConfig.Isolation = isolationLevel
w := tpcc.NewWorkloader(globalDB, &tpccConfig)

executeWorkload(globalCtx, w, action)
timeoutCtx, cancel := context.WithTimeout(globalCtx, totalTime)
defer cancel()

executeWorkload(timeoutCtx, w, action)
}

func registerTpcc(root *cobra.Command) {
Expand Down
6 changes: 5 additions & 1 deletion cmd/go-tpc/tpch.go
Original file line number Diff line number Diff line change
@@ -1,6 +1,7 @@
package main

import (
"context"
"strings"

"github.com/pingcap/go-tpc/tpch"
Expand All @@ -16,7 +17,10 @@ func executeTpch(action string, _ []string) {
tpchConfig.QueryNames = strings.Split(tpchConfig.RawQueries, ",")
w := tpch.NewWorkloader(globalDB, &tpchConfig)

executeWorkload(globalCtx, w, action)
timeoutCtx, cancel := context.WithTimeout(globalCtx, totalTime)
defer cancel()

executeWorkload(timeoutCtx, w, action)
}

func registerTpch(root *cobra.Command) {
Expand Down
6 changes: 6 additions & 0 deletions tpch/dbgen/README.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
dbgen
========

This dbgen was port from TPC-H dbgen v2.4.0.

Official TPC-H benchmark - [http://www.tpc.org/tpch](http://www.tpc.org/tpch)
2 changes: 2 additions & 0 deletions tpch/dbgen/copyright
Original file line number Diff line number Diff line change
@@ -0,0 +1,2 @@
TPC, TPC Benchmark and TPC-C are trademarks of the Transaction Processing Performance Council.
All other materials are ©2015-2016 TPC. All rights reserved.
83 changes: 83 additions & 0 deletions tpch/dbgen/cust.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
package dbgen

import (
"fmt"
"io"
)

const (
cPhneSd = 28
cAbalSd = 29
cMsegSd = 30
cAddrLen = 25
cCmntLen = 73
cAddrSd = 26
cCmntSd = 31
cAbalMin = -99999
cAbalMax = 999999
lNtrgSd = 27
)

type Cust struct {
CustKey dssHuge
Name string
Address string
NationCode dssHuge
Phone string
Acctbal dssHuge
MktSegment string
Comment string
}

type custLoader struct {
io.StringWriter
}

func (c custLoader) Load(item interface{}) error {
cust := item.(*Cust)
if _, err := c.WriteString(
fmt.Sprintf("%d|%s|%s|%d|%s|%s|%s|%s|\n",
cust.CustKey,
cust.Name,
cust.Address,
cust.NationCode,
cust.Phone,
FmtMoney(cust.Acctbal),
cust.MktSegment,
cust.Comment)); err != nil {
return err
}
return nil
}

func (c custLoader) Flush() error {
return nil
}

func newCustLoader(writer io.StringWriter) custLoader {
return custLoader{writer}
}

func sdCust(child Table, skipCount dssHuge) {
advanceStream(cAddrSd, skipCount*9, false)
advanceStream(cCmntSd, skipCount*2, false)
advanceStream(lNtrgSd, skipCount, false)
advanceStream(cPhneSd, skipCount*3, false)
advanceStream(cAbalSd, skipCount, false)
advanceStream(cMsegSd, skipCount, false)
}

func makeCust(idx dssHuge) *Cust {
cust := &Cust{}
cust.CustKey = idx
cust.Name = fmt.Sprintf("Customer#%09d", idx)
cust.Address = vStr(cAddrLen, cAddrSd)
i := random(0, dssHuge(nations.count-1), lNtrgSd)
cust.NationCode = i
cust.Phone = genPhone(i, cPhneSd)
cust.Acctbal = random(cAbalMin, cAbalMax, cAbalSd)
pickStr(&cMsegSet, cMsegSd, &cust.MktSegment)
cust.Comment = makeText(cCmntLen, cCmntSd)

return cust
}
92 changes: 92 additions & 0 deletions tpch/dbgen/dist.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
package dbgen

import "github.com/pingcap/go-tpc/tpch/dbgen/dist"

var (
nations distribution
nations2 distribution
regions distribution
oPrioritySet distribution
lInstructSet distribution
lSmodeSet distribution
lCategorySet distribution
lRflagSet distribution
cMsegSet distribution
colors distribution
pTypesSet distribution
pCntrSet distribution
articles distribution
nouns distribution
adjectives distribution
adverbs distribution
prepositions distribution
verbs distribution
terminators distribution
auxillaries distribution
np distribution
vp distribution
grammar distribution
)

type setMember struct {
weight long
text string
}

type distribution struct {
count int
max int32
members []setMember
permute []long
}

func readDist(name string, d *distribution) {
dist := dist.Maps[name]
d.count = len(dist)
for _, item := range dist {
d.max += item.Weight
d.members = append(d.members, setMember{text: item.Text, weight: long(d.max)})
}
}

func permute(permute []long, count int, stream long) {
for i := 0; i < count; i++ {
source := random(dssHuge(i), dssHuge(count-1), stream)
permute[source], permute[i] = permute[i], permute[source]
}
}

func permuteDist(dist *distribution, stream long) {
if len(dist.permute) == 0 {
dist.permute = make([]long, dist.count)
}
for i := 0; i < dist.count; i++ {
dist.permute[i] = long(i)
}
permute(dist.permute, dist.count, stream)
}

func initDists() {
readDist("p_cntr", &pCntrSet)
readDist("colors", &colors)
readDist("p_types", &pTypesSet)
readDist("nations", &nations)
readDist("regions", &regions)
readDist("o_oprio", &oPrioritySet)
readDist("instruct", &lInstructSet)
readDist("smode", &lSmodeSet)
readDist("category", &lCategorySet)
readDist("rflag", &lRflagSet)
readDist("msegmnt", &cMsegSet)
readDist("nouns", &nouns)
readDist("verbs", &verbs)
readDist("adjectives", &adjectives)
readDist("adverbs", &adverbs)
readDist("auxillaries", &auxillaries)
readDist("terminators", &terminators)
readDist("articles", &articles)
readDist("prepositions", &prepositions)
readDist("grammar", &grammar)
readDist("np", &np)
readDist("vp", &vp)
}
Loading

0 comments on commit b36483e

Please sign in to comment.