Ingest heads into explorer database

- Add to docker tools

core/store/migrations/migration1570675883/migrate.go

@@ -1,8 +1,8 @@
 package migration1570675883
 
 import (
-	"github.com/jinzhu/gorm"
 	"chainlink/core/store/models"
+	"github.com/jinzhu/gorm"
 )
 
 type JobRun struct {

go.sum

@@ -517,6 +517,7 @@ github.com/spf13/afero v1.2.1 h1:qgMbHoJbPbw579P+1zVY+6n4nIFuIchaIjzZ/I/Yq8M=
 github.com/spf13/afero v1.2.1/go.mod h1:9ZxEEn6pIJ8Rxe320qSDBk6AsU0r9pR7Q4OcevTdifk=
 github.com/spf13/cast v1.3.0 h1:oget//CVOEoFewqQxwr0Ej5yjygnqGkvggSE/gB35Q8=
 github.com/spf13/cast v1.3.0/go.mod h1:Qx5cxh0v+4UWYiBimWS+eyWzqEqokIECu5etghLkUJE=
+github.com/spf13/cobra v0.0.5 h1:f0B+LkLX6DtmRH1isoNA9VTtNUK9K8xYd28JNNfOv/s=
 github.com/spf13/cobra v0.0.5/go.mod h1:3K3wKZymM7VvHMDS9+Akkh4K60UwM26emMESw8tLCHU=
 github.com/spf13/jwalterweatherman v1.0.0 h1:XHEdyB+EcvlqZamSM4ZOMGlc93t6AcsBEu9Gc1vn7yk=
 github.com/spf13/jwalterweatherman v1.0.0/go.mod h1:cQK4TGJAtQXfYWX+Ddv3mKDzgVb68N+wFjFa4jdeBTo=

ingester/README.md

@@ -1,3 +1,26 @@
 # Chainlink Ingester
 
-This directory contains the ingester project, which is responsible for taking data off the chain and aggregating it for reporting or analysis purposes.
+This directory contains the ingester project, which is responsible for taking 
+data off the chain and aggregating it for reporting or analysis purposes.
+
+
+## Configuration
+
+The ingester application takes the following environment variables
+
+```bash
+# Ethereum Chain ID for the contracts you want to listen to
+ETH_CHAIN_ID
+# Websocket endpoint the monitor uses to watch the aggregator contracts
+ETH_URL
+# Postgres database host
+DB_HOST
+# Postgres database name
+DB_NAME
+# Postgres database port
+DB_PORT
+# Postgres database username
+DB_USERNAME
+# Postgres database password
+DB_PASSWORD
+```

ingester/service/application.go

@@ -33,15 +33,31 @@ func NewApplication(config *Config) (*Application, error) {
 	logger.Infow(
 		"Starting the Chainlink Ingester",
 		"eth-url", config.EthereumURL,
-		"db-url", config.DatabaseURL,
-		"eth-chain-id", config.NetworkID)
+		"eth-chain-id", config.NetworkID,
+		"db-host", config.DatabaseHost,
+		"db-name", config.DatabaseName,
+		"db-port", config.DatabasePort,
+		"db-username", config.DatabaseUsername,
+	)
 
 	ec, err := client.NewClient(config.EthereumURL)
 	if err != nil {
 		return nil, fmt.Errorf("Failed to connect to ETH client: %+v", err)
 	}
 
-	pool, err := sql.Open("postgres", config.DatabaseURL)
+	psqlInfo := fmt.Sprintf(
+		"host=%s port=%d user=%s password=%s dbname=%s sslmode=disable",
+		config.DatabaseHost,
+		config.DatabasePort,
+		config.DatabaseUsername,
+		config.DatabasePassword,
+		config.DatabaseName,
+	)
+	db, err := sql.Open("postgres", psqlInfo)
+	if err != nil {
+		return nil, err
+	}
+	err = db.Ping()
 	if err != nil {
 		return nil, err
 	}
@@ -65,7 +81,7 @@ func NewApplication(config *Config) (*Application, error) {
 			}
 
 			logger.Debugw("Oberved new log", "blockHash", log.BlockHash, "index", log.Index, "removed", log.Removed)
-			_, err := pool.Exec(`INSERT INTO "ethereum_log" ("address", "topics", "data", "blockNumber", "txHash", "txIndex", "blockHash", "index", "removed") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);`,
+			_, err := db.Exec(`INSERT INTO "ethereum_log" ("address", "topics", "data", "blockNumber", "txHash", "txIndex", "blockHash", "index", "removed") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9);`,
 				address,
 				topics,
 				log.Data,
@@ -94,7 +110,7 @@ func NewApplication(config *Config) (*Application, error) {
 			copy(nonce, head.Nonce[:])
 
 			logger.Debugw("Observed new head", "blockHeight", head.Number, "blockHash", head.Hash())
-			_, err := pool.Exec(`INSERT INTO "ethereum_head" ("blockHash", "parentHash", "uncleHash", "coinbase", "root", "txHash", "receiptHash", "bloom", "difficulty", "number", "gasLimit", "gasUsed", "time", "extra", "mixDigest", "nonce") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16);`,
+			_, err := db.Exec(`INSERT INTO "ethereum_head" ("blockHash", "parentHash", "uncleHash", "coinbase", "root", "txHash", "receiptHash", "bloom", "difficulty", "number", "gasLimit", "gasUsed", "time", "extra", "mixDigest", "nonce") VALUES ($1, $2, $3, $4, $5, $6, $7, $8, $9, $10, $11, $12, $13, $14, $15, $16);`,
 				head.Hash().Bytes(),
 				head.ParentHash,
 				head.UncleHash,

ingester/service/config.go

@@ -15,8 +15,16 @@ type Config struct {
 	NetworkID int `mapstructure:"eth-chain-id"`
 	// EthereumURL is the websocket endpoint the monitor uses to watch the aggregator contracts
 	EthereumURL string `mapstructure:"eth-url"`
-	// DatabaseURL is the url of the postgres server where the ingester saves results
-	DatabaseURL string `mapstructure:"db-url"`
+	// DatabaseHost of the postgres server where the ingester saves results
+	DatabaseHost string `mapstructure:"db-host"`
+	// DatabaseName of the postgres server where the ingester saves results
+	DatabaseName string `mapstructure:"db-name"`
+	// DatabasePort of the postgres server where the ingester saves results
+	DatabasePort int `mapstructure:"db-port"`
+	// DatabaseUsername of the postgres server where the ingester saves results
+	DatabaseUsername string `mapstructure:"db-username"`
+	// DatabasePassword of the postgres server where the ingester saves results
+	DatabasePassword string `mapstructure:"db-password"`
 }
 
 // NewConfig will return an instantiated config based on the passed in defaults
@@ -40,7 +48,11 @@ func DefaultConfig() *Config {
 		"response-timeout": time.Minute * 5,
 		"eth-chain-id":     1,
 		"eth-url":          "ws://localhost:8545",
-		"db-url":           "postgres://localhost:5432/explorer?sslmode=disable",
+		"db-host":          "localhost",
+		"db-name":          "explorer",
+		"db-port":          "5432",
+		"db-username":      "postgres",
+		"db-password":      "postgres",
 	})
 }
 

tools/docker/.env

@@ -23,13 +23,17 @@ SECURE_COOKIES=false
 
 # Explorer env vars
 EXPLORER_DB_NAME=explorer_dev
+EXPLORER_DB_PORT=5432
 EXPLORER_PGPASSWORD=explorer
 EXPLORER_SERVER_PORT=3001
 EXPLORER_COOKIE_SECRET=testingtestingtestingtestingtesting
 TYPEORM_DATABASE=explorer_dev
-TYPEORM_HOST=explorer-db
+TYPEORM_HOST=chainlink-explorer-db
 TYPEORM_USERNAME=postgres
 
+# Ingester
+INGESTER_DB_HOST=chainlink-explorer-db
+
 # Integration test env vars
 ETH_HTTP_URL=http://devnet:8545
 SRCROOT=/usr/local/src/chainlink

tools/docker/docker-compose.yaml

@@ -36,6 +36,15 @@ services:
       - apicredentials
       - keystore
 
+  explorer-db:
+    container_name: chainlink-explorer-db
+    image: postgres:11.6
+    volumes:
+      - explorer-db-data:/var/lib/postgresql/data
+    environment:
+      POSTGRES_DB: $EXPLORER_DB_NAME
+      POSTGRES_PASSWORD: $EXPLORER_PGPASSWORD
+
   explorer:
     container_name: chainlink-explorer
     image: chainlink/explorer
@@ -57,26 +66,22 @@ services:
       - PGPASSWORD=$EXPLORER_PGPASSWORD
 
   ingester:
-    container_name: ingester
+    container_name: chainlink-ingester
     image: chainlink/ingester
     build:
       context: ../../
       dockerfile: ingester/ingester.Dockerfile
     restart: always
+    depends_on:
+      - explorer-db
     environment:
       - ETH_CHAIN_ID
       - ETH_URL
-    depends_on:
-      - explorer
-
-  explorer-db:
-    container_name: chainlink-explorer-db
-    image: postgres:11.6
-    volumes:
-      - explorer-db-data:/var/lib/postgresql/data
-    environment:
-      POSTGRES_DB: $EXPLORER_DB_NAME
-      POSTGRES_PASSWORD: $EXPLORER_PGPASSWORD
+      - DB_HOST=$INGESTER_DB_HOST
+      - DB_NAME=$EXPLORER_DB_NAME
+      - DB_PORT=$EXPLORER_DB_PORT
+      - DB_USERNAME=$TYPEORM_USERNAME
+      - DB_PASSWORD=$EXPLORER_PGPASSWORD
 
 secrets:
   node_password: