Code restructuring to support V1alpha1 and V1alpha2 API (kubeflow#448)

* Code restructuring to support V1alpha1 and V1alpha2 API

* Adding comments

* Test package changes

* Moving requirements file

* Fix the package location

* Renaming studyjobcontroller to katib-controller

Makefile

@@ -0,0 +1,26 @@
+# Run tests
+test:
+	go test ./pkg/... ./cmd/...
+
+# Build Katib images
+build: 
+	sh scripts/build.sh
+
+# Deploy katib manifests into a k8s cluster
+deploy: 
+	sh scripts/deploy.sh
+
+# Run go fmt against code
+fmt:
+	go fmt ./pkg/... ./cmd/...
+
+# Run go vet against code
+vet:
+	go vet ./pkg/... ./cmd/...
+
+# Generate code
+generate:
+ifndef GOPATH
+	$(error GOPATH not defined, please define GOPATH. Run "go help gopath" to learn more about GOPATH)
+endif
+	go generate ./pkg/... ./cmd/...

cmd/earlystopping/medianstopping/Dockerfile

@@ -2,7 +2,7 @@ FROM golang:alpine AS build-env
 # The GOPATH in the image is /go.
 ADD . /go/src/github.com/kubeflow/katib
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/earlystopping/medianstopping
-RUN go build -o medianstopping
+RUN go build -o medianstopping ./v1alpha1
 
 FROM alpine:3.7
 WORKDIR /app

cmd/earlystopping/medianstopping/main.go → cmd/earlystopping/medianstopping/v1alpha1/main.go

@@ -7,8 +7,8 @@ import (
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/reflection"
 
-	pb "github.com/kubeflow/katib/pkg/api"
-	"github.com/kubeflow/katib/pkg/earlystopping"
+	pb "github.com/kubeflow/katib/pkg/api/v1alpha1"
+	earlystopping "github.com/kubeflow/katib/pkg/earlystopping/v1alpha1"
 )
 
 func main() {

cmd/katib-controller/Dockerfile

@@ -0,0 +1,16 @@
+# Build the manager binary
+FROM golang:alpine AS build-env
+
+# Copy in the go src
+ADD . /go/src/github.com/kubeflow/katib
+
+WORKDIR /go/src/github.com/kubeflow/katib/cmd/katib-controller
+# Build
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o studyjobcontroller  ./v1alpha1
+RUN CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -o katib-controller.v1alpha2  ./v1alpha2
+# Copy the controller-manager into a thin image
+FROM alpine:3.7
+WORKDIR /app
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/katib-controller/studyjobcontroller .
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/katib-controller/katib-controller.v1alpha2 .
+ENTRYPOINT ["./studyjobcontroller"]

cmd/studyjobcontroller/main.go → cmd/katib-controller/v1alpha1/main.go

@@ -26,7 +26,7 @@ import (
 	"log"
 
 	"github.com/kubeflow/katib/pkg/api/operators/apis"
-	"github.com/kubeflow/katib/pkg/controller"
+	controller "github.com/kubeflow/katib/pkg/controller/v1alpha1"
 	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
 	"sigs.k8s.io/controller-runtime/pkg/client/config"
 	"sigs.k8s.io/controller-runtime/pkg/manager"

cmd/katib-controller/v1alpha2/main.go

@@ -0,0 +1,69 @@
+/*
+Copyright 2018 The Kubeflow Authors
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+/*
+ StudyJobController is a controller (operator) for StudyJob
+ StudyJobController create and watch workers and metricscollectors.
+ The workers and metricscollectors are generated from template defined ConfigMap.
+ The workers and metricscollectors are kubernetes object. The default object is a Job and CronJob.
+*/
+package main
+
+import (
+	"log"
+
+	"github.com/kubeflow/katib/pkg/api/operators/apis"
+	controller "github.com/kubeflow/katib/pkg/controller/v1alpha2"
+	_ "k8s.io/client-go/plugin/pkg/client/auth/gcp"
+	"sigs.k8s.io/controller-runtime/pkg/client/config"
+	"sigs.k8s.io/controller-runtime/pkg/manager"
+	"sigs.k8s.io/controller-runtime/pkg/runtime/signals"
+)
+
+func main() {
+	// Get a config to talk to the apiserver
+	cfg, err := config.GetConfig()
+	if err != nil {
+		log.Printf("config.GetConfig()")
+		log.Fatal(err)
+	}
+
+	// Create a new StudyJobController to provide shared dependencies and start components
+	mgr, err := manager.New(cfg, manager.Options{})
+	if err != nil {
+		log.Printf("manager.New")
+		log.Fatal(err)
+	}
+
+	log.Printf("Registering Components.")
+
+	// Setup Scheme for all resources
+	if err := apis.AddToScheme(mgr.GetScheme()); err != nil {
+		log.Printf("apis.AddToScheme")
+		log.Fatal(err)
+	}
+
+	// Setup StudyJobController
+	if err := controller.AddToManager(mgr); err != nil {
+		log.Printf("controller.AddToManager(mgr)")
+		log.Fatal(err)
+	}
+
+	log.Printf("Starting the Cmd.")
+
+	// Starting the StudyJobController
+	log.Fatal(mgr.Start(signals.SetupSignalHandler()))
+}

cmd/manager-rest/Dockerfile

@@ -2,9 +2,11 @@ FROM golang:alpine AS build-env
 # The GOPATH in the image is /go.
 ADD . /go/src/github.com/kubeflow/katib
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/manager-rest
-RUN go build -o vizier-manager-rest
+RUN go build -o vizier-manager-rest ./v1alpha1
+RUN go build -o katib-manager-rest.v1alpha2 ./v1alpha2
 
 FROM alpine:3.7
 WORKDIR /app
 COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager-rest/vizier-manager-rest /app/
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager-rest/katib-manager-rest.v1alpha2 /app/
 ENTRYPOINT ["./vizier-manager-rest"]

cmd/manager-rest/v1alpha1/proxy.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/golang/glog"
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	gw "github.com/kubeflow/katib/pkg/api/v1alpha1"
+)
+
+var (
+	vizierCoreEndpoint = flag.String("echo_endpoint", "vizier-core:6789", "vizier-core endpoint")
+)
+
+func run() error {
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	mux := runtime.NewServeMux()
+	opts := []grpc.DialOption{grpc.WithInsecure()}
+	// register handlers for the HTTP endpoints
+	err := gw.RegisterManagerHandlerFromEndpoint(ctx, mux, *vizierCoreEndpoint, opts)
+	if err != nil {
+		return err
+	}
+
+	// proxy server listens on port 80
+	return http.ListenAndServe(":80", mux)
+}
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	if err := run(); err != nil {
+		glog.Fatal(err)
+	}
+}

cmd/manager-rest/v1alpha2/proxy.go

@@ -0,0 +1,43 @@
+package main
+
+import (
+	"flag"
+	"net/http"
+
+	"github.com/golang/glog"
+	"github.com/grpc-ecosystem/grpc-gateway/runtime"
+	"golang.org/x/net/context"
+	"google.golang.org/grpc"
+
+	gw "github.com/kubeflow/katib/pkg/api/v1alpha2"
+)
+
+var (
+	vizierCoreEndpoint = flag.String("echo_endpoint", "vizier-core:6789", "vizier-core endpoint")
+)
+
+func run() error {
+	ctx := context.Background()
+	ctx, cancel := context.WithCancel(ctx)
+	defer cancel()
+
+	mux := runtime.NewServeMux()
+	opts := []grpc.DialOption{grpc.WithInsecure()}
+	// register handlers for the HTTP endpoints
+	err := gw.RegisterManagerHandlerFromEndpoint(ctx, mux, *vizierCoreEndpoint, opts)
+	if err != nil {
+		return err
+	}
+
+	// proxy server listens on port 80
+	return http.ListenAndServe(":80", mux)
+}
+
+func main() {
+	flag.Parse()
+	defer glog.Flush()
+
+	if err := run(); err != nil {
+		glog.Fatal(err)
+	}
+}

cmd/manager/Dockerfile

@@ -2,14 +2,16 @@ FROM golang:alpine AS build-env
 # The GOPATH in the image is /go.
 ADD . /go/src/github.com/kubeflow/katib
 WORKDIR /go/src/github.com/kubeflow/katib/cmd/manager
-RUN go build -o vizier-manager
+RUN go build -o vizier-manager ./v1alpha1
+RUN go build -o katib-manager.v1alpha2 ./v1alpha2
 
 FROM alpine:3.7
 WORKDIR /app
 RUN GRPC_HEALTH_PROBE_VERSION=v0.2.0 && \
     wget -qO/bin/grpc_health_probe https://github.com/grpc-ecosystem/grpc-health-probe/releases/download/${GRPC_HEALTH_PROBE_VERSION}/grpc_health_probe-linux-amd64 && \
     chmod +x /bin/grpc_health_probe
 COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager/vizier-manager /app/
-COPY --from=build-env /go/src/github.com/kubeflow/katib/pkg/manager/visualise /
+COPY --from=build-env /go/src/github.com/kubeflow/katib/cmd/manager/katib-manager.v1alpha2 /app/
+#COPY --from=build-env /go/src/github.com/kubeflow/katib/pkg/manager/v1alpha1/visualise /
 ENTRYPOINT ["./vizier-manager"]
 CMD ["-w", "kubernetes"]

cmd/manager/main.go → cmd/manager/v1alpha1/main.go

@@ -9,9 +9,9 @@ import (
 	"net"
 	"time"
 
-	api_pb "github.com/kubeflow/katib/pkg/api"
-	health_pb "github.com/kubeflow/katib/pkg/api/health"
-	kdb "github.com/kubeflow/katib/pkg/db"
+	api_pb "github.com/kubeflow/katib/pkg/api/v1alpha1"
+	health_pb "github.com/kubeflow/katib/pkg/api/v1alpha1/health"
+	kdb "github.com/kubeflow/katib/pkg/db/v1alpha1"
 	"github.com/kubeflow/katib/pkg/manager/modelstore"
 
 	"google.golang.org/grpc"

cmd/manager/main_test.go → cmd/manager/v1alpha1/main_test.go

@@ -6,8 +6,8 @@ import (
 
 	"github.com/golang/mock/gomock"
 
-	api "github.com/kubeflow/katib/pkg/api"
-	mockdb "github.com/kubeflow/katib/pkg/mock/db"
+	api "github.com/kubeflow/katib/pkg/api/v1alpha1"
+	mockdb "github.com/kubeflow/katib/pkg/mock/v1alpha1/db"
 )
 
 func TestCreateStudy(t *testing.T) {