Skip to content

Commit

Permalink
Merge branch 'neural-network' into develop
Browse files Browse the repository at this point in the history 
Conflicts:
	neural_network.go
  • Loading branch information
@xlvector
xlvector committed Nov 30, 2013
2 parents 1d09eee + 931d3e3 commit 96283fc
Show file tree
Hide file tree
Showing 6 changed files with 113 additions and 49 deletions.
16 changes: 9 additions & 7 deletions bin/hector-cv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -35,13 +35,15 @@ func main(){
total := int(cv)

if profile != "" {
f, err := os.Create(profile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
fmt.Println(profile)
f, err := os.Create(profile)
if err != nil {
fmt.Println("%v", err)
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}

average_auc := 0.0
for part := 0; part < total; part++ {
Expand Down
16 changes: 8 additions & 8 deletions bin/hector-mc-cv.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -31,18 +31,18 @@ func main(){
profile, _ := params["profile"]
dataset := hector.NewDataSet()
dataset.Load(train_path, global)

cv, _ := strconv.ParseInt(params["cv"], 10, 32)
total := int(cv)

if profile != "" {
f, err := os.Create(profile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}
f, err := os.Create(profile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}

average_accuracy := 0.0
for part := 0; part < total; part++ {
Expand Down
14 changes: 14 additions & 0 deletions bin/hector-mc-run.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,9 @@ package main
import(
"hector"
"fmt"
"log"
"runtime/pprof"
"os"
)

func main(){
Expand All @@ -12,6 +15,17 @@ func main(){

classifier := hector.GetMutliClassClassifier(method)

profile, _ := params["profile"]
if profile != "" {
fmt.Printf("Profile data => %s\n", profile)
f, err := os.Create(profile)
if err != nil {
log.Fatal(err)
}
pprof.StartCPUProfile(f)
defer pprof.StopCPUProfile()
}

if action == "" {
accuracy, _ := hector.MultiClassRun(classifier, train, test, pred, params)
fmt.Println("accuracy : ", accuracy)
Expand Down
8 changes: 4 additions & 4 deletions matrix.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,12 @@ func (m *Matrix) AddValue(k1, k2 int64, v float64){
}

func (m *Matrix) SetValue(k1, k2 int64, v float64){
_, ok := m.data[k1]
row, ok := m.data[k1]
if !ok {
m.data[k1] = NewVector()
row = NewVector()
m.data[k1] = row
}
m.data[k1].SetValue(k2, v)
row.SetValue(k2, v)
}

func (m *Matrix) GetValue(k1, k2 int64) float64 {
Expand All @@ -42,7 +43,6 @@ func (m *Matrix) GetRow(k1 int64) *Vector {
} else {
return row
}
return row
}

func (m *Matrix) Scale(scale float64) *Matrix {
Expand Down
104 changes: 74 additions & 30 deletions neural_network.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -9,9 +9,11 @@ import(

type NeuralNetworkParams struct {
LearningRate float64
LearningRateDiscount float64
Regularization float64
Hidden int64
Steps int
Verbose int
}

type TwoLayerWeights struct {
Expand All @@ -26,7 +28,7 @@ http://www4.rgu.ac.uk/files/chapter3%20-%20bp.pdf
*/
type NeuralNetwork struct {
Model TwoLayerWeights
MaxLabel int
MaxLabel int64
Params NeuralNetworkParams
}

Expand All @@ -49,18 +51,22 @@ func (self *NeuralNetwork) LoadModel(path string){

func (algo *NeuralNetwork) Init(params map[string]string) {
algo.Params.LearningRate, _ = strconv.ParseFloat(params["learning-rate"], 64)
algo.Params.LearningRateDiscount, _ = strconv.ParseFloat(params["learning-rate-discount"], 64)
algo.Params.Regularization, _ = strconv.ParseFloat(params["regularization"], 64)
steps, _ := strconv.ParseInt(params["steps"], 10, 32)
hidden, _ := strconv.ParseInt(params["hidden"], 10, 64)
verbose, _ := strconv.ParseInt(params["verbose"], 10, 32)

algo.Params.Steps = int(steps)
algo.Params.Hidden = int64(hidden)
algo.Params.Verbose = int(verbose)
}

func (algo *NeuralNetwork) Train(dataset * DataSet) {
algo.Model = TwoLayerWeights{}

algo.Model.L1 = NewMatrix()
algo.Model.L2 = NewMatrix()

for i := int64(0); i < algo.Params.Hidden; i++ {
algo.Model.L1.data[i] = NewVector()
}
Expand All @@ -81,57 +87,76 @@ func (algo *NeuralNetwork) Train(dataset * DataSet) {
}
}
}
algo.MaxLabel = max_label
algo.MaxLabel = int64(max_label)

algo.Model.L2 = NewMatrix()
for i := int64(0); i < algo.Params.Hidden; i++ {
for j := int64(0); j <= int64(max_label); j++ {
algo.Model.L2.SetValue(i, j, (rand.NormFloat64() / math.Sqrt(float64(max_label) + 1.0)))
for i := int64(0); i <= algo.Params.Hidden; i++ {
for j := int64(0); j < algo.MaxLabel; j++ {
algo.Model.L2.SetValue(i, j, (rand.NormFloat64() / math.Sqrt(float64(algo.MaxLabel) + 1.0)))
}
}

for step := 0; step < algo.Params.Steps; step++{
fmt.Printf(".")
if algo.Params.Verbose <= 0 {
fmt.Printf(".")
}
total := len(dataset.Samples)
counter := 0
for _, sample := range dataset.Samples {
y := NewVector()
z := NewVector()
e := NewVector()
delta_hidden := NewVector()

for i := int64(0); i < algo.Params.Hidden; i++ {
sum := float64(0)
wi := algo.Model.L1.data[i]
for _, f := range sample.Features {
sum += f.Value * algo.Model.L1.data[i].GetValue(f.Id)
sum += f.Value * wi.GetValue(f.Id)
}
y.data[i] = Sigmoid(sum)
for j := int64(0); j <= int64(max_label); j++ {
z.AddValue(j, y.GetValue(i) * algo.Model.L2.GetValue(i, j))
}
y.data[algo.Params.Hidden] = 1.0
for i := int64(0); i < algo.MaxLabel; i++ {
sum := float64(0)
for j := int64(0); j <= algo.Params.Hidden; j++ {
sum += y.GetValue(j)*algo.Model.L2.GetValue(j, i)
}
z.SetValue(i, sum)
}
z = z.SoftMaxNorm()
e.SetValue(int64(sample.Label), 1.0)
e.AddVector(z, -1.0)

err := NewVector()
err.AddValue(int64(sample.Label), 1.0)
err.AddVector(z, -1.0)

delta_hidden := NewVector()
for i := int64(0); i < algo.Params.Hidden; i++ {
for j := int64(0); j <= int64(max_label); j++ {
for i := int64(0); i <= algo.Params.Hidden; i++ {
delta := float64(0)
for j := int64(0); j < algo.MaxLabel; j++ {
wij := algo.Model.L2.GetValue(i, j)
delta_j := err.GetValue(j)

sig_ij := delta_j * (1-z.GetValue(j)) * z.GetValue(j)
delta_hidden.AddValue(i, sig_ij * wij)
sig_ij := e.GetValue(j) * (1-z.GetValue(j)) * z.GetValue(j)
delta += sig_ij * wij
wij += algo.Params.LearningRate * (y.GetValue(i) * sig_ij - algo.Params.Regularization * wij)
algo.Model.L2.SetValue(i, j, wij)
}
delta_hidden.SetValue(i, delta)
}

for _, f := range sample.Features {
for j := int64(0); j < algo.Params.Hidden; j++ {
wij := algo.Model.L1.GetValue(j, f.Id)
wij += algo.Params.LearningRate * (delta_hidden.GetValue(j) * f.Value * y.GetValue(j) * (1-y.GetValue(j)) - algo.Params.Regularization * wij)
algo.Model.L1.SetValue(j, f.Id, wij)
for i := int64(0); i < algo.Params.Hidden; i++ {
wi := algo.Model.L1.data[i]
for _, f := range sample.Features {
wji := wi.GetValue(f.Id)
wji += algo.Params.LearningRate * (delta_hidden.GetValue(i) * f.Value * y.GetValue(i) * (1-y.GetValue(i)) - algo.Params.Regularization * wji)
wi.SetValue(f.Id, wji)
}
}
counter++
if algo.Params.Verbose > 0 && counter % 2000 == 0 {
fmt.Printf("Epoch %d %f%%\n", step+1, float64(counter)/float64(total)*100)
}
}

if algo.Params.Verbose > 0 {
algo.Evaluate(dataset)
}
algo.Params.LearningRate *= algo.Params.LearningRateDiscount
}
fmt.Println()
}
Expand All @@ -145,9 +170,14 @@ func (algo *NeuralNetwork) PredictMultiClass(sample * Sample) * ArrayVector {
sum += f.Value * algo.Model.L1.data[i].GetValue(f.Id)
}
y.data[i] = Sigmoid(sum)
for j := 0; j <= algo.MaxLabel; j++ {
z.AddValue(j, y.GetValue(i) * algo.Model.L2.GetValue(i, int64(j)))
}
y.data[algo.Params.Hidden] = 1
for i := 0; i < int(algo.MaxLabel); i++ {
sum := float64(0)
for j := int64(0); j <= algo.Params.Hidden; j++ {
sum += y.GetValue(j) * algo.Model.L2.GetValue(j, int64(i))
}
z.SetValue(i, sum)
}
z = z.SoftMaxNorm()
fmt.Println(z)
Expand All @@ -157,4 +187,18 @@ func (algo *NeuralNetwork) PredictMultiClass(sample * Sample) * ArrayVector {
func (algo *NeuralNetwork) Predict(sample *Sample) float64 {
z := algo.PredictMultiClass(sample)
return z.GetValue(1)
}
}

func (algo *NeuralNetwork) Evaluate(dataset *DataSet) {
accuracy := 0.0
total := 0.0
for _, sample := range dataset.Samples {
prediction := algo.PredictMultiClass(sample)
label, _ := prediction.KeyWithMaxValue()
if int(label) == sample.Label {
accuracy += 1.0
}
total += 1.0
}
fmt.Printf("accuracy %f%%\n", accuracy/total*100)
}
4 changes: 4 additions & 0 deletions params.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -73,7 +73,9 @@ func PrepareParams() (string, string, string, string, map[string]string){
test_path := flag.String("test", "test.tsv", "path of testing file")
pred_path := flag.String("pred", "", "path of pred file")
output := flag.String("output", "", "output file path")
verbose := flag.Int("v", 0, "verbose output if 1")
learning_rate := flag.String("learning-rate", "0.01", "learning rate")
learning_rate_discount := flag.String("learning-rate-discount", "1.0", "discount rate of learning rate per training step")
regularization := flag.String("regularization", "0.01", "regularization")
alpha := flag.String("alpha", "0.1", "alpha of ftrl")
beta := flag.String("beta", "1", "beta of ftrl")
Expand Down Expand Up @@ -106,7 +108,9 @@ func PrepareParams() (string, string, string, string, map[string]string){
fmt.Println(*train_path)
fmt.Println(*test_path)
fmt.Println(*method)
params["verbose"] = strconv.FormatInt(int64(*verbose), 10)
params["learning-rate"] = *learning_rate
params["learning-rate-discount"] = *learning_rate_discount
params["regularization"] = *regularization
params["alpha"] = *alpha
params["beta"] = *beta
Expand Down

0 comments on commit 96283fc

Please sign in to comment.