build: improvements & upgrades to Nix dev environment

* Add some more useful stuff to the shell environment, so it actually works for debugging & post-mortem analysis. * Wrap ccache & distcc transparently (distcc will be used unless NODISTCC is set to a non-empty value in the environment; ccache will be used if CCACHE_DIR is not empty). * Package the Scylla Python driver (instead of the C* one). * Catch up to misc build/test requirements (including optional) by requiring or custom-packaging: wasmtime 0.29.0, cxxbridge, pytest-asyncio, liburing. * Build statically-linked zstd in a saner and more idiomatic fashion. * In pure builds (where sources lack Git metadata), derive SCYLLA_RELEASE from source hash. * Refactor things for more parameterization. * Explicitly stub out installPhase (seeing that "nix build" succeeds up to installPhase means we didn't miss any dependencies). * Add flake support. * Add copious comments. Signed-off-by: Michael Livshin <[email protected]>
sarvex · Oct 2, 2022 · 7bd13be · 7bd13be
1 parent 839d8f4
commit 7bd13be
Show file tree

Hide file tree

Showing 11 changed files with 907 additions and 132 deletions.
diff --git a/configure.py b/configure.py
@@ -44,32 +44,18 @@
 distro_extra_ldflags = ''
 distro_extra_cmake_args = []
 employ_ld_trickery = True
+has_wasmtime = False
+use_wasmtime_as_library = False
 
 # distro-specific setup
 def distro_setup_nix():
-    global os_ids, employ_ld_trickery
-    global distro_extra_ldflags, distro_extra_cflags, distro_extra_cmake_args
-
+    global os_ids, employ_ld_trickery, has_wasmtime, use_wasmtime_as_library
     os_ids = ['linux']
     employ_ld_trickery = False
+    has_wasmtime = True
+    use_wasmtime_as_library = True
 
-    libdirs = list(dict.fromkeys(os.environ.get('CMAKE_LIBRARY_PATH').split(':')))
-    incdirs = list(dict.fromkeys(os.environ.get('CMAKE_INCLUDE_PATH').split(':')))
-
-    # add nix {lib,inc}dirs to relevant flags, mimicing nix versions of cmake & autotools.
-    # also add rpaths to make sure that any built executables can run in place.
-    distro_extra_ldflags = ' '.join([ '-rpath ' + path + ' -L' + path for path in libdirs ]);
-    distro_extra_cflags = ' '.join([ '-isystem ' + path for path in incdirs ])
-
-    # indexers like clangd may or may not know which stdc++ or glibc
-    # the compiler was configured with, so make the relevant paths
-    # explicit on each compilation command line:
-    implicit_cflags = os.environ.get('IMPLICIT_CFLAGS').strip()
-    distro_extra_cflags += ' ' + implicit_cflags
-    # also propagate to cmake-built dependencies:
-    distro_extra_cmake_args = ['-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES:INTERNAL=' + implicit_cflags]
-
-if os.environ.get('NIX_BUILD_TOP'):
+if os.environ.get('NIX_CC'):
         distro_setup_nix()
 
 # distribution "internationalization", converting package names.
@@ -1423,7 +1409,8 @@ def clang_inline_threshold():
     for mode in modes:
         modes[mode]['cxxflags'] += f' -Wstack-usage={modes[mode]["stack-usage-threshold"]} -Wno-error=stack-usage='
 
-has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime')
+if not has_wasmtime:
+    has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime')
 
 if has_wasmtime:
     if platform.machine() == 'aarch64':
@@ -1755,6 +1742,8 @@ def configure_abseil(build_dir, mode, mode_config):
                 ])
 if has_wasmtime:
     print("Found wasmtime dependency, linking with libwasmtime")
+    if use_wasmtime_as_library:
+        libs += " -lwasmtime"
 
 if not args.staticboost:
     args.user_cflags += ' -DBOOST_TEST_DYN_LINK'
@@ -1936,7 +1925,7 @@ def configure_abseil(build_dir, mode, mode_config):
                     for src in srcs
                     if src.endswith('.cc')]
             objs.append('$builddir/../utils/arch/powerpc/crc32-vpmsum/crc32.S')
-            if has_wasmtime:
+            if has_wasmtime and not use_wasmtime_as_library:
                 objs.append('/usr/lib64/libwasmtime.a')
             has_thrift = False
             for dep in deps[binary]:

diff --git a/default.nix b/default.nix
@@ -6,83 +6,122 @@
 #
 
 #
-# * At present this is not very useful for nix-build, just for nix-shell
+# * "nix build" is unsupported (or rather supported up to and not including
+#   installPhase), so basically all this is just for "nix develop"
 #
 # * IMPORTANT: to avoid using up ungodly amounts of disk space under
-#   /nix/store/, make sure the actual build directory is physically
-#   outside this tree, and make ./build a symlink to it
+#   /nix/store/ when you are not using flakes, make sure to move the
+#   actual build directory outside this tree and make ./build a
+#   symlink to it.  Or use flakes (seriously, just use flakes).
 #
 
-{
-  pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/4cb48cc25622334f17ec6b9bf56e83de0d521fb7.tar.gz") {},
-  mode ? "release",
-  verbose ? false,
-  useCcache ? false, # can't get this to work, see https://github.com/NixOS/nixpkgs/issues/49894
-  testInputsFrom ? (_: []),
-  gitPkg ? (pkgs: pkgs.gitMinimal),
-}:
+{ flake ? false
+, shell ? false
+, pkgs ? import <nixpkgs> { system = builtins.currentSystem; overlays = [ (import ./dist/nix/overlay.nix <nixpkgs>) ]; }
+, srcPath ? builtins.path { path = ./.; name = "scylla"; }
+, repl ? null
+, mode ? "release"
+, verbose ? false
 
-with pkgs; let
+# shell env will want to add stuff to the environment, and the way
+# for it to do so is to pass us a function with this signatire:
+, devInputs ? ({ pkgs, llvm }: [])
+} @ args:
+
+let
   inherit (builtins)
     baseNameOf
     fetchurl
+    head
+    map
     match
-    readFile
-    toString
-    trace;
-
-  antlr3Patched = antlr3.overrideAttrs (_: {
-    patches = [
-      (fetchurl {
-        url = "https://src.fedoraproject.org/rpms/antlr3/raw/f1bb8d639678047935e1761c3bf3c1c7da8d0f1d/f/0006-antlr3memory.hpp-fix-for-C-20-mode.patch";
-      })
-    ];
-  });
-  rapidjsonPatched = rapidjson.overrideAttrs (_: {
-    patches = [
-      (fetchurl {
-        url = "https://src.fedoraproject.org/rpms/rapidjson/raw/48402da9f19d060ffcd40bf2b2e6987212c58b0c/f/rapidjson-1.1.0-c++20.patch";
-      })
-    ];
-  });
-  zstdStatic = zstd.overrideAttrs (_: {
-    cmakeFlags = [
-      "-DZSTD_BUILD_SHARED:BOOL=OFF"
-      "-DZSTD_BUILD_STATIC:BOOL=ON"
-      "-DZSTD_PROGRAMS_LINK_SHARED:BOOL=OFF"
-      "-DZSTD_LEGACY_SUPPORT:BOOL=ON"
-      "-DZSTD_BUILD_TESTS:BOOL=OFF"
-    ];
-  });
-
-  llvmBundle = llvmPackages_11;
-
-  stdenv =
-    if useCcache
-    then (overrideCC llvmBundle.stdenv (ccacheWrapper.override { cc = llvmBundle.clang; }))
-    else llvmBundle.stdenv;
-
-in stdenv.mkDerivation {
-  name = "scylladb";
-  nativeBuildInputs = [
+    split
+    trace
+  ;
+
+  inherit (import (builtins.fetchTarball {
+    url = "https://github.com/hercules-ci/gitignore/archive/5b9e0ff9d3b551234b4f3eb3983744fa354b17f1.tar.gz";
+    sha256 = "01l4phiqgw9xgaxr6jr456qmww6kzghqrnbc7aiiww3h6db5vw53";
+  }) { inherit (pkgs) lib; })
+    gitignoreSource;
+
+  # tests don't like boost17x (which is boost177 at the time of writing)
+  boost = pkgs.boost175;
+
+  # current clang13 cannot compile Scylla with sanitizers:
+  llvm = pkgs.llvmPackages_12;
+  # llvm = pkgs.llvmPackages_latest;
+
+  stdenvUnwrapped = llvm.stdenv;
+
+  # define custom ccache- and distcc-aware wrappers for all relevant
+  # compile drivers (used only in shell env)
+  cc-wrappers = pkgs.callPackage ./dist/nix/pkg/custom/ccache-distcc-wrap {
+    cc = stdenvUnwrapped.cc;
+    clang = llvm.clang;
+    inherit (pkgs) gcc;
+  };
+
+  stdenv = if shell then pkgs.overrideCC stdenvUnwrapped cc-wrappers
+           else stdenvUnwrapped;
+
+  noNix = path: type: type != "regular" || (match ".*\.nix" path) == null;
+  src = builtins.filterSource noNix (if flake then srcPath
+                                     else gitignoreSource srcPath);
+
+  derive = if shell then pkgs.mkShell.override { inherit stdenv; }
+           else stdenv.mkDerivation;
+
+in derive ({
+  name = "scylla";
+  inherit src;
+
+  # since Scylla build, as it exists, is not cross-capable, the
+  # nativeBuildInputs/buildInputs distinction below ranges, depending
+  # on how charitable one feels, from "pedantic" through
+  # "aspirational" all the way to "cargo cult ritual" -- i.e. not
+  # expected to be actually correct or verifiable.  but it's the
+  # thought that counts!
+  nativeBuildInputs = with pkgs; [
     ant
-    antlr3Patched
-    boost17x.dev
+    antlr3
+    boost
+    cargo
     cmake
+    cxxbridge
     gcc
-    (gitPkg pkgs)
+    openjdk11_headless
     libtool
-    llvmBundle.lld
+    llvm.bintools
     maven
     ninja
     pkg-config
-    python3
+    python2
+    (python3.withPackages (ps: with ps; [
+      aiohttp
+      boto3
+      colorama
+      distro
+      magic
+      psutil
+      pyparsing
+      pytest
+      pytest-asyncio
+      pyudev
+      pyyaml
+      requests
+      scylla-driver
+      setuptools
+      tabulate
+      urwid
+    ]))
     ragel
     stow
-  ];
-  buildInputs = [
-    antlr3Patched
-    boost17x
+  ] ++ (devInputs { inherit pkgs llvm; });
+
+  buildInputs = with pkgs; [
+    antlr3
+    boost
     c-ares
     cryptopp
     fmt
@@ -91,13 +130,16 @@ in stdenv.mkDerivation {
     hwloc
     icu
     jsoncpp
+    libidn2
     libp11
     libsystemtap
     libtasn1
     libunistring
+    liburing
     libxfs
     libxml2
     libyamlcpp
+    llvm.compiler-rt
     lksctp-tools
     lua53Packages.lua
     lz4
@@ -106,61 +148,57 @@ in stdenv.mkDerivation {
     openssl
     p11-kit
     protobuf
-    python3Packages.cassandra-driver
-    python3Packages.distro
-    python3Packages.psutil
-    python3Packages.pyparsing
-    python3Packages.pyudev
-    python3Packages.pyyaml
-    python3Packages.requests
-    python3Packages.setuptools
-    python3Packages.urwid
-    rapidjsonPatched
+    rapidjson
     snappy
     systemd
     thrift
     valgrind
+    wasmtime
     xorg.libpciaccess
     xxHash
     zlib
     zstdStatic
-  ] ++ (testInputsFrom pkgs);
-
-  src = lib.cleanSourceWith {
-    filter = name: type:
-      let baseName = baseNameOf (toString name); in
-      !((type == "symlink" && baseName == "build") ||
-        (type == "directory" &&
-         (baseName == "build" ||
-          baseName == ".cache" ||
-          baseName == ".direnv" ||
-          baseName == ".github" ||
-          baseName == ".pytest_cache" ||
-          baseName == "__pycache__")));
-    src = ./.;
-  };
+  ];
+
+  JAVA8_HOME = "${pkgs.openjdk8_headless}/lib/openjdk";
+  JAVA_HOME = "${pkgs.openjdk11_headless}/lib/openjdk";
+
+}
+// (if shell then {
+
+  configurePhase = "./configure.py${if verbose then " --verbose" else ""} --disable-dpdk";
+
+} else {
+
+  # sha256 of the filtered source tree:
+  SCYLLA_RELEASE = head (split "-" (baseNameOf src));
 
   postPatch = ''
     patchShebangs ./configure.py
-    patchShebangs ./merge-compdb.py
     patchShebangs ./seastar/scripts/seastar-json2code.py
     patchShebangs ./seastar/cooking.sh
   '';
 
-  IMPLICIT_CFLAGS = ''
-    ${readFile (llvmBundle.stdenv.cc + "/nix-support/libcxx-cxxflags")} ${readFile (llvmBundle.stdenv.cc + "/nix-support/libc-cflags")}
-  '';
-
-  configurePhase = ''
-    ./configure.py ${if verbose then "--verbose " else ""}--mode=${mode}
-  '';
+  configurePhase = "./configure.py${if verbose then " --verbose" else ""} --mode=${mode}";
 
   buildPhase = ''
-    ${ninja}/bin/ninja build/${mode}/scylla
+    ${pkgs.ninja}/bin/ninja \
+      build/${mode}/scylla \
+      build/${mode}/iotune \
+
   '';
+   #   build/${mode}/dist/tar/scylla-tools-package.tar.gz \
+   #   build/${mode}/dist/tar/scylla-jmx-package.tar.gz \
 
   installPhase = ''
-    mkdir $out
-    cp -r * $out/
+    echo not implemented 1>&2
+    exit 1
   '';
-}
+
+})
+// (if !shell || repl == null then {} else {
+
+  REPL = repl;
+
+})
+)
diff --git a/dist/nix/overlay.nix b/dist/nix/overlay.nix
@@ -0,0 +1,25 @@
+nixpkgs:
+final: prev:
+
+let
+  patched = pkg: patches:
+    pkg.overrideAttrs (old: {
+      patches = (old.patches or []) ++ (map final.fetchurl patches);
+    });
+in {
+  gdbWithGreenThreadSupport = patched prev.gdb [{
+    url = "https://github.com/cmm/gnu-binutils/commit/1c52ca4b27e93e1684c68eeaee44ca3e36648410.patch";
+    sha256 = "sha256-3s3KvN70dHMdr7Sx1dtzbZ8S+MynPTN7yCocoGlea2Y=";
+  }];
+
+  zstdStatic = final.callPackage "${nixpkgs}/pkgs/tools/compression/zstd" {
+    static = true;
+    buildContrib = false;
+    doCheck = false;
+  };
+
+  cxxbridge = final.callPackage ./pkg/upstreamable/cxxbridge { };
+  wasmtime = final.callPackage ./pkg/upstreamable/wasmtime { };
+
+  scylla-driver = final.callPackage ./pkg/upstreamable/python-driver { };
+}