Jemalloc for develop (MinaProtocol#3662)

DoniLiMay · Oct 17, 2019 · 7591939 · 7591939
1 parent d43bd36
commit 7591939
Show file tree

Hide file tree

Showing 13 changed files with 92 additions and 50 deletions.
diff --git a/.circleci/config.yml b/.circleci/config.yml
@@ -61,7 +61,7 @@ jobs:
 
     build-archive:
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
               environment: 
                 CODA_DOCKER: true
                 HASURA_PORT: 8080
@@ -91,7 +91,7 @@ jobs:
                   command: ls && ./scripts/archive/make_hasura_configurations.sh     
     lint:
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -262,7 +262,7 @@ jobs:
     build-artifacts--testnet_postake_medium_curves:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -311,7 +311,7 @@ jobs:
     test-unit--test_postake_snarkless_unittest:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run: ulimit -c unlimited
@@ -349,7 +349,7 @@ jobs:
     test-unit--dev:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run: ulimit -c unlimited
@@ -387,7 +387,7 @@ jobs:
     test-unit--dev_medium_curves:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run: ulimit -c unlimited
@@ -403,7 +403,7 @@ jobs:
     test--fake_hash:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -417,7 +417,7 @@ jobs:
     test--test_postake:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -434,7 +434,7 @@ jobs:
     test--test_postake_bootstrap:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -451,7 +451,7 @@ jobs:
     test--test_postake_catchup:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -465,7 +465,7 @@ jobs:
     test--test_postake_delegation:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -479,7 +479,7 @@ jobs:
     test--test_postake_five_even_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -493,7 +493,7 @@ jobs:
     test--test_postake_five_even_txns:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -507,7 +507,7 @@ jobs:
     test--test_postake_holy_grail:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -524,7 +524,7 @@ jobs:
     test--test_postake_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -541,7 +541,7 @@ jobs:
     test--test_postake_split:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -555,7 +555,7 @@ jobs:
     test--test_postake_split_snarkless:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -590,7 +590,7 @@ jobs:
     test--test_postake_three_proposers:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -604,7 +604,7 @@ jobs:
     test--test_postake_txns:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -621,7 +621,7 @@ jobs:
     test--test_postake_medium_curves:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -640,7 +640,7 @@ jobs:
     test--test_postake_snarkless_medium_curves:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -657,7 +657,7 @@ jobs:
     test--test_postake_split_medium_curves:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:

diff --git a/.circleci/config.yml.jinja b/.circleci/config.yml.jinja
@@ -61,7 +61,7 @@ jobs:
 
     build-archive:
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
               environment: 
                 CODA_DOCKER: true
                 HASURA_PORT: 8080
@@ -91,7 +91,7 @@ jobs:
                   command: ls && ./scripts/archive/make_hasura_configurations.sh     
     lint:
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -268,7 +268,7 @@ jobs:
         resource_class: large
         {%- endif %}
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -326,7 +326,7 @@ jobs:
     test-unit--{{profile}}:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run: ulimit -c unlimited
@@ -367,7 +367,7 @@ jobs:
     test-unit--{{profile}}:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run: ulimit -c unlimited
@@ -386,7 +386,7 @@ jobs:
     test--{{profile}}:
         resource_class: large
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:
@@ -405,7 +405,7 @@ jobs:
     test--{{profile}}:
         resource_class: xlarge
         docker:
-            - image: codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+            - image: codaprotocol/coda:toolchain-03e9ed5771ebca91003ce50fb246640f3d5ff061
         steps:
             - checkout
             - run:

diff --git a/README-dev.md b/README-dev.md
@@ -57,7 +57,7 @@ of the repo.
 
 * Pull down developer container image  (~2GB download, go stretch your legs)
 
-`docker pull codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17`
+`docker pull codaprotocol/coda:toolchain-FIXME`
 
 * Create local builder image
 

diff --git a/dockerfiles/Dockerfile b/dockerfiles/Dockerfile
@@ -1,4 +1,4 @@
-FROM codaprotocol/coda:toolchain-0b4ea6c4f8e5ba17bf5965dca7c74759442bff17
+FROM codaprotocol/coda:toolchain-FIXME
 
 # same as in Dockerfile-toolchain
 ARG OCAML_VERSION=4.07.1

diff --git a/dockerfiles/Dockerfile-toolchain b/dockerfiles/Dockerfile-toolchain
@@ -17,6 +17,7 @@ RUN sudo apt-get update && sudo apt-get install --yes \
     libffi-dev \
     libgmp-dev \
     libgmp3-dev \
+    libjemalloc-dev \
     libprocps-dev \
     libsodium-dev \
     libssl-dev \

diff --git a/scripts/macos-setup-brew.sh b/scripts/macos-setup-brew.sh
@@ -3,7 +3,7 @@ set -eu
 
 export HOMEBREW_NO_AUTO_UPDATE=1
 
-NEEDED_PACKAGES="gpatch opam cmake gmp pkg-config openssl libffi libsodium boost zlib libomp bash"
+NEEDED_PACKAGES="bash boost cmake gmp gpatch jemalloc libffi libomp libsodium opam openssl pkg-config zlib"
 echo "Needed:  ${NEEDED_PACKAGES}"
 
 CURRENT_PACKAGES=$(brew list | xargs)

diff --git a/scripts/rebuild-deb.sh b/scripts/rebuild-deb.sh
@@ -32,7 +32,7 @@ Version: ${VERSION}
 Section: base
 Priority: optional
 Architecture: amd64
-Depends: libssl1.1, libprocps6, libgmp10, libffi6, libgomp1, miniupnpc, coda-discovery
+Depends: coda-discovery, libffi6, libgmp10, libgomp1, libjemalloc1, libprocps6, libssl1.1, miniupnpc
 License: Apache-2.0
 Homepage: https://codaprotocol.com/
 Maintainer: o(1)Labs <[email protected]>
@@ -116,7 +116,7 @@ Version: ${VERSION}
 Section: base
 Priority: optional
 Architecture: amd64
-Depends: libssl1.1, libprocps6, libgmp10, libffi6, libgomp1, miniupnpc, coda-discovery
+Depends: coda-discovery, libffi6, libgmp10, libgomp1, libjemalloc1, libprocps6, libssl1.1, miniupnpc
 License: Apache-2.0
 Homepage: https://codaprotocol.com/
 Maintainer: o(1)Labs <[email protected]>

diff --git a/src/app/cli/src/coda.ml b/src/app/cli/src/coda.ml
@@ -386,7 +386,17 @@ let daemon logger =
                 ; ("heap_chunks", `Int stat.heap_chunks)
                 ; ("max_heap_size", `Int (stat.top_heap_words * bytes_per_word))
                 ; ("live_size", `Int (stat.live_words * bytes_per_word))
-                ; ("live_blocks", `Int stat.live_blocks) ]
+                ; ("live_blocks", `Int stat.live_blocks) ] ;
+            let {Jemalloc.active; resident; allocated; mapped} =
+              Jemalloc.get_memory_stats ()
+            in
+            Logger.debug logger ~module_:__MODULE__ ~location:__LOC__
+              "Jemalloc memory statistics (in bytes)"
+              ~metadata:
+                [ ("active", `Int (active * 1024))
+                ; ("resident", `Int (resident * 1024))
+                ; ("allocated", `Int (allocated * 1024))
+                ; ("mapped", `Int (mapped * 1024)) ]
           in
           let rec loop () =
             log_stats "before major gc" ;

diff --git a/src/app/cli/src/dune b/src/app/cli/src/dune
@@ -3,7 +3,7 @@
   (name coda)
   (public_name coda)
   (modes native)
-  (libraries init tests child_processes)
+  (libraries init tests child_processes jemalloc)
   (preprocessor_deps ../../../config.mlh)
   (preprocess (pps ppx_coda ppx_let ppx_sexp_conv ppx_optcomp))
   (flags -short-paths -g -w @a-4-29-40-41-42-44-45-48-58-59-60))
diff --git a/src/dune b/src/dune
@@ -5,4 +5,4 @@
   (action (copy config/config.mlh config.mlh)))
 
 (env
-  (_ (flags (:standard -short-paths -w @a-4-29-40-41-42-44-45-48-58-59-60))))
+  (_ (flags (:standard -short-paths -cclib -ljemalloc -w @a-4-29-40-41-42-44-45-48-58-59-60))))