Skip to content

Commit

Permalink
concurrent I/O per drive yields higher throughput
Browse files Browse the repository at this point in the history
4 is a sufficient number for concurrent I/O per
drive to reach maximum throughput, allow this to
be default behavior
  • Loading branch information
harshavardhana committed May 27, 2022
1 parent 723adb1 commit 19bb72b
Show file tree
Hide file tree
Showing 3 changed files with 55 additions and 21 deletions.
64 changes: 49 additions & 15 deletions pkg/dperf/perf.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"os"
"path/filepath"
"sort"
"strconv"
"sync"

"github.com/google/uuid"
Expand All @@ -44,29 +45,62 @@ func mustGetUUID() string {
return u.String()
}

const ioPerDrive = 4 // number of concurrent I/O per drive

func (d *DrivePerf) runTests(ctx context.Context, path string, testUUID string) (dr *DrivePerfResult) {
writeThroughputs := make([]uint64, ioPerDrive)
readThroughputs := make([]uint64, ioPerDrive)
errs := make([]error, ioPerDrive)

testUUIDPath := filepath.Join(path, testUUID)
testPath := filepath.Join(testUUIDPath, ".writable-check.tmp")
defer os.RemoveAll(testUUIDPath)

writeThroughput, err := d.runWriteTest(ctx, testPath)
if err != nil {
return &DrivePerfResult{
Path: path,
Error: err,
}

var wg sync.WaitGroup
wg.Add(ioPerDrive)
for i := 0; i < ioPerDrive; i++ {
go func(idx int) {
defer wg.Done()
iopath := testPath + "-" + strconv.Itoa(idx)
writeThroughput, err := d.runWriteTest(ctx, iopath)
if err != nil {
errs[idx] = err
return
}
writeThroughputs[idx] = writeThroughput
readThroughput, err := d.runReadTest(ctx, iopath)
if err != nil {
errs[idx] = err
return
}
readThroughputs[idx] = readThroughput
}(i)
}
readThroughput, err := d.runReadTest(ctx, testPath)
if err != nil {
return &DrivePerfResult{
Path: path,
Error: err,
wg.Wait()

for _, err := range errs {
if err != nil {
return &DrivePerfResult{
Path: path,
Error: err,
}
}
}

var writeThroughput uint64
for i := range writeThroughputs {
writeThroughput += writeThroughputs[i]
}

var readThroughput uint64
for i := range readThroughputs {
readThroughput += readThroughputs[i]
}

return &DrivePerfResult{
Path: path,
ReadThroughput: uint64(readThroughput),
WriteThroughput: uint64(writeThroughput),
ReadThroughput: readThroughput,
WriteThroughput: writeThroughput,
}
}

Expand Down Expand Up @@ -101,7 +135,7 @@ func (d *DrivePerf) Run(ctx context.Context, paths ...string) ([]*DrivePerfResul
if childCtx.Err() != nil {
return nil, childCtx.Err()
}

return results, nil
}

Expand Down
8 changes: 4 additions & 4 deletions pkg/dperf/run_linux.go
Original file line number Diff line number Diff line change
Expand Up @@ -93,7 +93,7 @@ func (o *odirectReader) Close() error {
return o.File.Close()
}

func (d *DrivePerf) runReadTest(ctx context.Context, path string) (float64, error) {
func (d *DrivePerf) runReadTest(ctx context.Context, path string) (uint64, error) {
startTime := time.Now()
f, err := directio.OpenFile(path, os.O_RDONLY, 0400)
if err != nil {
Expand All @@ -119,7 +119,7 @@ func (d *DrivePerf) runReadTest(ctx context.Context, path string) (float64, erro
of.Close()

timeTakenInSeconds := time.Since(startTime).Seconds()
return float64(d.FileSize) / timeTakenInSeconds, nil
return d.FileSize / uint64(timeTakenInSeconds), nil
}

// disableDirectIO - disables directio mode.
Expand Down Expand Up @@ -169,7 +169,7 @@ func (n nullReader) Read(b []byte) (int, error) {
return len(b), nil
}

func (d *DrivePerf) runWriteTest(ctx context.Context, path string) (float64, error) {
func (d *DrivePerf) runWriteTest(ctx context.Context, path string) (uint64, error) {
if err := os.MkdirAll(filepath.Dir(path), 0o755); err != nil {
return 0, err
}
Expand Down Expand Up @@ -199,5 +199,5 @@ func (d *DrivePerf) runWriteTest(ctx context.Context, path string) (float64, err
sync()
timeTakenInSeconds := time.Since(startTime).Seconds()

return float64(d.FileSize) / timeTakenInSeconds, nil
return d.FileSize / uint64(timeTakenInSeconds), nil
}
4 changes: 2 additions & 2 deletions pkg/dperf/run_other.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,10 +21,10 @@ package dperf

import "context"

func (d *DrivePerf) runReadTest(ctx context.Context, path string) (float64, error) {
func (d *DrivePerf) runReadTest(ctx context.Context, path string) (uint64, error) {
return 0, ErrNotImplemented
}

func (d *DrivePerf) runWriteTest(ctx context.Context, path string) (float64, error) {
func (d *DrivePerf) runWriteTest(ctx context.Context, path string) (uint64, error) {
return 0, ErrNotImplemented
}

0 comments on commit 19bb72b

Please sign in to comment.