Skip to content

Commit

Permalink
refactor
Browse files Browse the repository at this point in the history
  • Loading branch information
@xitongsys
xitongsys committed May 24, 2019
1 parent 66f2822 commit 7b208cb
Show file tree
Hide file tree
Showing 56 changed files with 515 additions and 448 deletions.
68 changes: 68 additions & 0 deletions Gopkg.lock
View file

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion Common/Common.go → common/common.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package Common
package common

import (
"bytes"
Expand Down
4 changes: 2 additions & 2 deletions Common/Common_test.go → common/common_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package Common
package common

import (
"bytes"
Expand All @@ -7,7 +7,7 @@ import (
"reflect"
"testing"

. "github.com/xitongsys/parquet-go/ParquetType"
. "github.com/xitongsys/parquet-go/types"
"github.com/xitongsys/parquet-go/parquet"
)

Expand Down
2 changes: 1 addition & 1 deletion Compress/Compress.go → compress/compress.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package Compress
package compress

import (
"bytes"
Expand Down
2 changes: 1 addition & 1 deletion ParquetEncoding/BinaryRead.go → encoding/binaryread.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ParquetEncoding
package encoding

import (
"io"
Expand Down
2 changes: 1 addition & 1 deletion ParquetEncoding/BinaryWrite.go → encoding/binarywrite.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ParquetEncoding
package encoding

import (
"io"
Expand Down
2 changes: 1 addition & 1 deletion ParquetEncoding/EncodingRead.go → encoding/encodingread.go
View file
Original file line number Diff line number Diff line change
@@ -1,4 +1,4 @@
package ParquetEncoding
package encoding

import (
"bytes"
Expand Down
4 changes: 2 additions & 2 deletions ParquetEncoding/EncodingRead_test.go → encoding/encodingread_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,11 +1,11 @@
package ParquetEncoding
package encoding

import (
"bytes"
"fmt"
"testing"

. "github.com/xitongsys/parquet-go/Common"
. "github.com/xitongsys/parquet-go/common"
"github.com/xitongsys/parquet-go/parquet"
)

Expand Down
10 changes: 5 additions & 5 deletions ParquetEncoding/EncodingWrite.go → encoding/encodingwrite.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package ParquetEncoding
package encoding

import (
"bytes"
"reflect"

"github.com/xitongsys/parquet-go/Common"
"github.com/xitongsys/parquet-go/common"
"github.com/xitongsys/parquet-go/parquet"
)

Expand Down Expand Up @@ -121,7 +121,7 @@ func WritePlainFIXED_LEN_BYTE_ARRAY(arrays []interface{}) []byte {
}

func WriteUnsignedVarInt(num uint64) []byte {
byteNum := (Common.BitNum(uint64(num)) + 6) / 7
byteNum := (common.BitNum(uint64(num)) + 6) / 7
if byteNum == 0 {
return make([]byte, 1)
}
Expand Down Expand Up @@ -288,7 +288,7 @@ func WriteDeltaINT32(nums []interface{}) []byte {
maxValue = blockBuf[k].(int32)
}
}
bitWidths[j] = byte(Common.BitNum(uint64(maxValue)))
bitWidths[j] = byte(common.BitNum(uint64(maxValue)))
}

var minDeltaZigZag uint64 = uint64((minDelta >> 31) ^ (minDelta << 1))
Expand Down Expand Up @@ -346,7 +346,7 @@ func WriteDeltaINT64(nums []interface{}) []byte {
maxValue = blockBuf[k].(int64)
}
}
bitWidths[j] = byte(Common.BitNum(uint64(maxValue)))
bitWidths[j] = byte(common.BitNum(uint64(maxValue)))
}

var minDeltaZigZag uint64 = uint64((minDelta >> 63) ^ (minDelta << 1))
Expand Down
4 changes: 2 additions & 2 deletions ParquetEncoding/EncodingWrite_test.go → encoding/encodingwrite_test.go
View file
Original file line number Diff line number Diff line change
@@ -1,10 +1,10 @@
package ParquetEncoding
package encoding

import (
"encoding/json"
"testing"

. "github.com/xitongsys/parquet-go/Common"
. "github.com/xitongsys/parquet-go/common"
"github.com/xitongsys/parquet-go/parquet"
)

Expand Down
14 changes: 7 additions & 7 deletions example/column_read.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@ import (
"log"
"time"

"github.com/xitongsys/parquet-go/ParquetFile"
"github.com/xitongsys/parquet-go/ParquetReader"
"github.com/xitongsys/parquet-go/ParquetWriter"
"github.com/xitongsys/parquet-go/source"
"github.com/xitongsys/parquet-go/reader"
"github.com/xitongsys/parquet-go/writer"
)

type Student struct {
Expand All @@ -23,12 +23,12 @@ type Student struct {
func main() {
var err error
//write
fw, err := ParquetFile.NewLocalFileWriter("column.parquet")
fw, err := source.NewLocalFileWriter("column.parquet")
if err != nil {
log.Println("Can't create file", err)
return
}
pw, err := ParquetWriter.NewParquetWriter(fw, new(Student), 4)
pw, err := writer.NewParquetWriter(fw, new(Student), 4)
if err != nil {
log.Println("Can't create parquet writer")
return
Expand Down Expand Up @@ -59,12 +59,12 @@ func main() {
var rls, dls []int32

///read
fr, err := ParquetFile.NewLocalFileReader("column.parquet")
fr, err := source.NewLocalFileReader("column.parquet")
if err != nil {
log.Println("Can't open file", err)
return
}
pr, err := ParquetReader.NewParquetColumnReader(fr, 4)
pr, err := reader.NewParquetColumnReader(fr, 4)
if err != nil {
log.Println("Can't create column reader", err)
return
Expand Down
8 changes: 4 additions & 4 deletions example/csv_write.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -4,8 +4,8 @@ import (
"fmt"
"log"

"github.com/xitongsys/parquet-go/ParquetFile"
"github.com/xitongsys/parquet-go/ParquetWriter"
"github.com/xitongsys/parquet-go/source"
"github.com/xitongsys/parquet-go/writer"
)

func main() {
Expand All @@ -19,12 +19,12 @@ func main() {
}

//write
fw, err := ParquetFile.NewLocalFileWriter("csv.parquet")
fw, err := source.NewLocalFileWriter("csv.parquet")
if err != nil {
log.Println("Can't open file", err)
return
}
pw, err := ParquetWriter.NewCSVWriter(md, fw, 4)
pw, err := writer.NewCSVWriter(md, fw, 4)
if err != nil {
log.Println("Can't create csv writer", err)
return
Expand Down
19 changes: 9 additions & 10 deletions example/gcs_write.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -5,9 +5,8 @@ import (
"fmt"
"log"

"github.com/xitongsys/parquet-go/ParquetFile"
"github.com/xitongsys/parquet-go/ParquetType"
"github.com/xitongsys/parquet-go/ParquetWriter"
"github.com/xitongsys/parquet-go/source"
"github.com/xitongsys/parquet-go/writer"
)

func main() {
Expand All @@ -25,12 +24,12 @@ func main() {
fileName := "gcs_example/csv.parquet"

//write
fw, err := ParquetFile.NewGcsFileWriter(ctx, projectId, bucketName, fileName)
fw, err := source.NewGcsFileWriter(ctx, projectId, bucketName, fileName)
if err != nil {
log.Println("Can't open file", err)
return
}
pw, err := ParquetWriter.NewCSVWriter(md, fw, 4)
pw, err := writer.NewCSVWriter(md, fw, 4)
if err != nil {
log.Println("Can't create csv writer", err)
return
Expand All @@ -54,11 +53,11 @@ func main() {
}

data2 := []interface{}{
ParquetType.BYTE_ARRAY("Student Name"),
ParquetType.INT32(20 + i%5),
ParquetType.INT64(i),
ParquetType.FLOAT(50.0 + float32(i)*0.1),
ParquetType.BOOLEAN(i%2 == 0),
[]byte("Student Name"),
int32(20 + i%5),
int64(i),
float32(50.0 + float32(i)*0.1),
bool(i%2 == 0),
}
if err = pw.Write(data2); err != nil {
log.Println("Write error", err)
Expand Down
14 changes: 7 additions & 7 deletions example/hdfs_flat.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,9 +3,9 @@ package main
import (
"log"

"github.com/xitongsys/parquet-go/ParquetFile"
"github.com/xitongsys/parquet-go/ParquetReader"
"github.com/xitongsys/parquet-go/ParquetWriter"
"github.com/xitongsys/parquet-go/source"
"github.com/xitongsys/parquet-go/reader"
"github.com/xitongsys/parquet-go/writer"
)

type Student struct {
Expand All @@ -19,12 +19,12 @@ type Student struct {
func main() {
var err error
//write
fw, err := ParquetFile.NewHdfsFileWriter([]string{"localhost:9000"}, "root", "/flat.parquet")
fw, err := source.NewHdfsFileWriter([]string{"localhost:9000"}, "root", "/flat.parquet")
if err != nil {
log.Println("Can't create hdfs file", err)
return
}
pw, err := ParquetWriter.NewParquetWriter(fw, new(Student), 4)
pw, err := writer.NewParquetWriter(fw, new(Student), 4)
if err != nil {
log.Println("Can't create parquet writer", err)
return
Expand All @@ -50,12 +50,12 @@ func main() {
fw.Close()

///read
fr, err := ParquetFile.NewHdfsFileReader([]string{"localhost:9000"}, "", "/flat.parquet")
fr, err := source.NewHdfsFileReader([]string{"localhost:9000"}, "", "/flat.parquet")
if err != nil {
log.Println("Can't open hdfs file", err)
return
}
pr, err := ParquetReader.NewParquetReader(fr, new(Student), 4)
pr, err := reader.NewParquetReader(fr, new(Student), 4)
if err != nil {
log.Println("Can't create parquet reader", err)
return
Expand Down
Loading

0 comments on commit 7b208cb

Please sign in to comment.