[QUEUED scylladb next] Merge 'Upgrade nix devenv' from Michael Livshin

[Commit Bot]

From: Botond Dénes <bde...@scylladb.com>
Committer: Botond Dénes <bde...@scylladb.com>
Branch: next

Merge 'Upgrade nix devenv' from Michael Livshin

To recap: the Nix devenv ({default,shell,flake}.nix and friends) in Scylla is a nicer (for those who consider it so, that is) alternative to dbuild: a completely deterministic build environment without Docker.

In theory we could support much more (creating installable packages, container images, various deployment affordances, etc. -- Nix is, among other things, a kind of parallel-to-everything-else devops realm) but there is clearly no demand and besides duplicating the work the release team is already doing (and doing just fine, needless to say) would be pointless and wasteful.

This PR reflects the accumulated changes that I have been carrying locally for the past year or so. The version currently in master _probably_ can still build Scylla, but that Scylla certainly would not pass unit tests.

What the previous paragraph seems to mean is, apparently I'm the only active user of Nix devenv for Scylla. Which, in turn, presents some obvious questions for the maintainers:

- Does this need to live in the Scylla source at all? (The changes to non-Nix-specific parts are minimal and unobtrusive, but they are still changes)
- If it's left in, who is going to maintain it going forward, should more users somehow appear? (I'm perfectly willing to fix things up when alerted, but no timeliness guarantees)

Closes #9557

* github.com:scylladb/scylladb:
nix: add README.md
build: improvements & upgrades to Nix dev environment
build: allow setting SCYLLA_RELEASE from outside

---
diff --git a/.gitignore b/.gitignore
--- a/.gitignore
+++ b/.gitignore
@@ -31,3 +31,4 @@ docs/poetry.lock
compile_commands.json
.ccls-cache/
.mypy_cache
+.envrc
diff --git a/SCYLLA-VERSION-GEN b/SCYLLA-VERSION-GEN
--- a/SCYLLA-VERSION-GEN
+++ b/SCYLLA-VERSION-GEN
@@ -79,14 +79,20 @@ then
SCYLLA_VERSION=$(cat version | awk -F'-' '{print $1}')
SCYLLA_RELEASE=$(cat version | awk -F'-' '{print $2}')
else
- GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
SCYLLA_VERSION=$VERSION
- # For custom package builds, replace "0" with "counter.your_name",
- # where counter starts at 1 and increments for successive versions.
- # This ensures that the package manager will select your custom
- # package over the standard release.
- SCYLLA_BUILD=0
- SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
+ if [ -z "$SCYLLA_RELEASE" ]; then
+ DATE=$(date --utc +%Y%m%d)
+ GIT_COMMIT=$(git -C "$SCRIPT_DIR" log --pretty=format:'%h' -n 1 --abbrev=12)
+ # For custom package builds, replace "0" with "counter.your_name",
+ # where counter starts at 1 and increments for successive versions.
+ # This ensures that the package manager will select your custom
+ # package over the standard release.
+ SCYLLA_BUILD=0
+ SCYLLA_RELEASE=$SCYLLA_BUILD.$DATE.$GIT_COMMIT
+ elif [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
+ echo "setting SCYLLA_RELEASE only makes sense in clean builds" 1>&2
+ exit 1
+ fi
fi

if [ -f "$OUTPUT_DIR/SCYLLA-RELEASE-FILE" ]; then
diff --git a/configure.py b/configure.py
--- a/configure.py
+++ b/configure.py
@@ -44,32 +44,18 @@
distro_extra_ldflags = ''
distro_extra_cmake_args = []
employ_ld_trickery = True
+has_wasmtime = False
+use_wasmtime_as_library = False

# distro-specific setup
def distro_setup_nix():
- global os_ids, employ_ld_trickery
- global distro_extra_ldflags, distro_extra_cflags, distro_extra_cmake_args
-
+ global os_ids, employ_ld_trickery, has_wasmtime, use_wasmtime_as_library
os_ids = ['linux']
employ_ld_trickery = False
+ has_wasmtime = True
+ use_wasmtime_as_library = True

- libdirs = list(dict.fromkeys(os.environ.get('CMAKE_LIBRARY_PATH').split(':')))
- incdirs = list(dict.fromkeys(os.environ.get('CMAKE_INCLUDE_PATH').split(':')))
-
- # add nix {lib,inc}dirs to relevant flags, mimicing nix versions of cmake & autotools.
- # also add rpaths to make sure that any built executables can run in place.
- distro_extra_ldflags = ' '.join([ '-rpath ' + path + ' -L' + path for path in libdirs ]);
- distro_extra_cflags = ' '.join([ '-isystem ' + path for path in incdirs ])
-
- # indexers like clangd may or may not know which stdc++ or glibc
- # the compiler was configured with, so make the relevant paths
- # explicit on each compilation command line:
- implicit_cflags = os.environ.get('IMPLICIT_CFLAGS').strip()
- distro_extra_cflags += ' ' + implicit_cflags
- # also propagate to cmake-built dependencies:
- distro_extra_cmake_args = ['-DCMAKE_CXX_STANDARD_INCLUDE_DIRECTORIES:INTERNAL=' + implicit_cflags]
-
-if os.environ.get('NIX_BUILD_TOP'):
+if os.environ.get('NIX_CC'):
distro_setup_nix()

# distribution "internationalization", converting package names.
@@ -1423,7 +1409,8 @@ def clang_inline_threshold():
for mode in modes:
modes[mode]['cxxflags'] += f' -Wstack-usage={modes[mode]["stack-usage-threshold"]} -Wno-error=stack-usage='

-has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime')
+if not has_wasmtime:
+ has_wasmtime = os.path.isfile('/usr/lib64/libwasmtime.a') and os.path.isdir('/usr/local/include/wasmtime')

if has_wasmtime:
if platform.machine() == 'aarch64':
@@ -1755,6 +1742,8 @@ def configure_abseil(build_dir, mode, mode_config):
])
if has_wasmtime:
print("Found wasmtime dependency, linking with libwasmtime")
+ if use_wasmtime_as_library:
+ libs += " -lwasmtime"

if not args.staticboost:
args.user_cflags += ' -DBOOST_TEST_DYN_LINK'
@@ -1936,7 +1925,7 @@ def configure_abseil(build_dir, mode, mode_config):
for src in srcs
if src.endswith('.cc')]
objs.append('$builddir/../utils/arch/powerpc/crc32-vpmsum/crc32.S')
- if has_wasmtime:
+ if has_wasmtime and not use_wasmtime_as_library:
objs.append('/usr/lib64/libwasmtime.a')
has_thrift = False
for dep in deps[binary]:
diff --git a/default.nix b/default.nix
--- a/default.nix
+++ b/default.nix
@@ -6,83 +6,122 @@
#

#
-# * At present this is not very useful for nix-build, just for nix-shell
+# * "nix build" is unsupported (or rather supported up to and not including
+# installPhase), so basically all this is just for "nix develop"
#
# * IMPORTANT: to avoid using up ungodly amounts of disk space under
-# /nix/store/, make sure the actual build directory is physically
-# outside this tree, and make ./build a symlink to it
+# /nix/store/ when you are not using flakes, make sure to move the
+# actual build directory outside this tree and make ./build a
+# symlink to it. Or use flakes (seriously, just use flakes).
#

-{
- pkgs ? import (fetchTarball "https://github.com/NixOS/nixpkgs/archive/4cb48cc25622334f17ec6b9bf56e83de0d521fb7.tar.gz") {},
- mode ? "release",
- verbose ? false,
- useCcache ? false, # can't get this to work, see https://github.com/NixOS/nixpkgs/issues/49894
- testInputsFrom ? (_: []),
- gitPkg ? (pkgs: pkgs.gitMinimal),
-}:
+{ flake ? false
+, shell ? false
+, pkgs ? import <nixpkgs> { system = builtins.currentSystem; overlays = [ (import ./dist/nix/overlay.nix <nixpkgs>) ]; }
+, srcPath ? builtins.path { path = ./.; name = "scylla"; }
+, repl ? null
+, mode ? "release"
+, verbose ? false

-with pkgs; let
+# shell env will want to add stuff to the environment, and the way
+# for it to do so is to pass us a function with this signatire:
+, devInputs ? ({ pkgs, llvm }: [])
+} @ args:
+
+let
inherit (builtins)
baseNameOf
fetchurl
+ head
+ map
match
- readFile
- toString
- trace;
-
- antlr3Patched = antlr3.overrideAttrs (_: {
- patches = [
- (fetchurl {
- url = "https://src.fedoraproject.org/rpms/antlr3/raw/f1bb8d639678047935e1761c3bf3c1c7da8d0f1d/f/0006-antlr3memory.hpp-fix-for-C-20-mode.patch";
- })
- ];
- });
- rapidjsonPatched = rapidjson.overrideAttrs (_: {
- patches = [
- (fetchurl {
- url = "https://src.fedoraproject.org/rpms/rapidjson/raw/48402da9f19d060ffcd40bf2b2e6987212c58b0c/f/rapidjson-1.1.0-c++20.patch";
- })
- ];
- });
- zstdStatic = zstd.overrideAttrs (_: {
- cmakeFlags = [
- "-DZSTD_BUILD_SHARED:BOOL=OFF"
- "-DZSTD_BUILD_STATIC:BOOL=ON"
- "-DZSTD_PROGRAMS_LINK_SHARED:BOOL=OFF"
- "-DZSTD_LEGACY_SUPPORT:BOOL=ON"
- "-DZSTD_BUILD_TESTS:BOOL=OFF"
- ];
- });
-
- llvmBundle = llvmPackages_11;
-
- stdenv =
- if useCcache
- then (overrideCC llvmBundle.stdenv (ccacheWrapper.override { cc = llvmBundle.clang; }))
- else llvmBundle.stdenv;
-
-in stdenv.mkDerivation {
- name = "scylladb";
- nativeBuildInputs = [
+ split
+ trace
+ ;
+
+ inherit (import (builtins.fetchTarball {
+ url = "https://github.com/hercules-ci/gitignore/archive/5b9e0ff9d3b551234b4f3eb3983744fa354b17f1.tar.gz";
+ sha256 = "01l4phiqgw9xgaxr6jr456qmww6kzghqrnbc7aiiww3h6db5vw53";
+ }) { inherit (pkgs) lib; })
+ gitignoreSource;
+
+ # tests don't like boost17x (which is boost177 at the time of writing)
+ boost = pkgs.boost175;
+
+ # current clang13 cannot compile Scylla with sanitizers:
+ llvm = pkgs.llvmPackages_12;
+ # llvm = pkgs.llvmPackages_latest;
+
+ stdenvUnwrapped = llvm.stdenv;
+
+ # define custom ccache- and distcc-aware wrappers for all relevant
+ # compile drivers (used only in shell env)
+ cc-wrappers = pkgs.callPackage ./dist/nix/pkg/custom/ccache-distcc-wrap {
+ cc = stdenvUnwrapped.cc;
+ clang = llvm.clang;
+ inherit (pkgs) gcc;
+ };
+
+ stdenv = if shell then pkgs.overrideCC stdenvUnwrapped cc-wrappers
+ else stdenvUnwrapped;
+
+ noNix = path: type: type != "regular" || (match ".*\.nix" path) == null;
+ src = builtins.filterSource noNix (if flake then srcPath
+ else gitignoreSource srcPath);
+
+ derive = if shell then pkgs.mkShell.override { inherit stdenv; }
+ else stdenv.mkDerivation;
+
+in derive ({
+ name = "scylla";
+ inherit src;
+
+ # since Scylla build, as it exists, is not cross-capable, the
+ # nativeBuildInputs/buildInputs distinction below ranges, depending
+ # on how charitable one feels, from "pedantic" through
+ # "aspirational" all the way to "cargo cult ritual" -- i.e. not
+ # expected to be actually correct or verifiable. but it's the
+ # thought that counts!
+ nativeBuildInputs = with pkgs; [
ant
- antlr3Patched
- boost17x.dev
+ antlr3
+ boost
+ cargo
cmake
+ cxxbridge
gcc
- (gitPkg pkgs)
+ openjdk11_headless
libtool
- llvmBundle.lld
+ llvm.bintools
maven
ninja
pkg-config
- python3
+ python2
+ (python3.withPackages (ps: with ps; [
+ aiohttp
+ boto3
+ colorama
+ distro
+ magic
+ psutil
+ pyparsing
+ pytest
+ pytest-asyncio
+ pyudev
+ pyyaml
+ requests
+ scylla-driver
+ setuptools
+ tabulate
+ urwid
+ ]))
ragel
stow
- ];
- buildInputs = [
- antlr3Patched
- boost17x
+ ] ++ (devInputs { inherit pkgs llvm; });
+
+ buildInputs = with pkgs; [
+ antlr3
+ boost
c-ares
cryptopp
fmt
@@ -91,13 +130,16 @@ in stdenv.mkDerivation {
hwloc
icu
jsoncpp
+ libidn2
libp11
libsystemtap
libtasn1
libunistring
+ liburing
libxfs
libxml2
libyamlcpp
+ llvm.compiler-rt
lksctp-tools
lua53Packages.lua
lz4
@@ -106,61 +148,57 @@ in stdenv.mkDerivation {
openssl
p11-kit
protobuf
- python3Packages.cassandra-driver
- python3Packages.distro
- python3Packages.psutil
- python3Packages.pyparsing
- python3Packages.pyudev
- python3Packages.pyyaml
- python3Packages.requests
- python3Packages.setuptools
- python3Packages.urwid
- rapidjsonPatched
+ rapidjson
snappy
systemd
thrift
valgrind
+ wasmtime
xorg.libpciaccess
xxHash
zlib
zstdStatic
- ] ++ (testInputsFrom pkgs);
-
- src = lib.cleanSourceWith {
- filter = name: type:
- let baseName = baseNameOf (toString name); in
- !((type == "symlink" && baseName == "build") ||
- (type == "directory" &&
- (baseName == "build" ||
- baseName == ".cache" ||
- baseName == ".direnv" ||
- baseName == ".github" ||
- baseName == ".pytest_cache" ||
- baseName == "__pycache__")));
- src = ./.;
- };
+ ];
+
+ JAVA8_HOME = "${pkgs.openjdk8_headless}/lib/openjdk";
+ JAVA_HOME = "${pkgs.openjdk11_headless}/lib/openjdk";
+
+}
+// (if shell then {
+
+ configurePhase = "./configure.py${if verbose then " --verbose" else ""} --disable-dpdk";
+
+} else {
+
+ # sha256 of the filtered source tree:
+ SCYLLA_RELEASE = head (split "-" (baseNameOf src));

postPatch = ''
patchShebangs ./configure.py
- patchShebangs ./merge-compdb.py
patchShebangs ./seastar/scripts/seastar-json2code.py
patchShebangs ./seastar/cooking.sh
'';

- IMPLICIT_CFLAGS = ''
- ${readFile (llvmBundle.stdenv.cc + "/nix-support/libcxx-cxxflags")} ${readFile (llvmBundle.stdenv.cc + "/nix-support/libc-cflags")}
- '';
-
- configurePhase = ''
- ./configure.py ${if verbose then "--verbose " else ""}--mode=${mode}
- '';
+ configurePhase = "./configure.py${if verbose then " --verbose" else ""} --mode=${mode}";

buildPhase = ''
- ${ninja}/bin/ninja build/${mode}/scylla
+ ${pkgs.ninja}/bin/ninja \
+ build/${mode}/scylla \
+ build/${mode}/iotune \
+
'';
+ # build/${mode}/dist/tar/scylla-tools-package.tar.gz \
+ # build/${mode}/dist/tar/scylla-jmx-package.tar.gz \

installPhase = ''
- mkdir $out
- cp -r * $out/
+ echo not implemented 1>&2
+ exit 1
'';
-}
+
+})
+// (if !shell || repl == null then {} else {
+
+ REPL = repl;
+
+})
+)
diff --git a/dist/nix/README.md b/dist/nix/README.md
--- a/dist/nix/README.md
+++ b/dist/nix/README.md
@@ -0,0 +1,42 @@
+# Nix development environment
+
+The `*.nix` files at the project root and under `dist/nix` implement a
+[Nix](https://nixos.org/guides/ad-hoc-developer-environments.html)-based
+development environment for Scylla.
+
+Note that there is presently no support for building installable
+artifacts with `nix build`, or anything suitable for incorporation
+into the `Nixpkgs` collection (that's "just" a matter of implementing
+`installPhase` in `default.nix`). This is just a development
+environment that is predictable, is independent from the state of the
+host distribution, and does not require entering a container.
+
+`gdb` with green thread debugging support is included, plus other
+assorted debugging tools.
+
+Compilers are transparently wrapped to use `ccache` and `distcc`, if
+you have those configured.
+
+## Basic usage
+
+If you have Nix flake support enabled (recommended!), enter the
+environment using `nix develop .`. Otherwise, use `nix-shell .`.
+
+`$configPhase` will configure Scylla for building (it just invokes
+`./configure.py --disable-dpdk`). Then use `ninja` to build,
+`test.py` to run unit tests, etc., as usual.
+
+## Direnv (optional)
+
+Using [direnv](https://direnv.net) is recommended to make life more convenient. Create
+`.envrc` at the project root with the contents:
+```bash
+nix_direnv_watch_file default.nix
+nix_direnv_watch_file shell.nix
+for f in $(find dist/nix/ -type f); do
+ nix_direnv_watch_file $f
+done
+use flake
+```
+
+For non-flake usage, replace the last line with `use nix`.
diff --git a/dist/nix/overlay.nix b/dist/nix/overlay.nix
--- a/dist/nix/overlay.nix
+++ b/dist/nix/overlay.nix
@@ -0,0 +1,25 @@
+nixpkgs:
+final: prev:
+
+let
+ patched = pkg: patches:
+ pkg.overrideAttrs (old: {
+ patches = (old.patches or []) ++ (map final.fetchurl patches);
+ });
+in {
+ gdbWithGreenThreadSupport = patched prev.gdb [{
+ url = "https://github.com/cmm/gnu-binutils/commit/1c52ca4b27e93e1684c68eeaee44ca3e36648410.patch";
+ sha256 = "sha256-3s3KvN70dHMdr7Sx1dtzbZ8S+MynPTN7yCocoGlea2Y=";
+ }];
+
+ zstdStatic = final.callPackage "${nixpkgs}/pkgs/tools/compression/zstd" {
+ static = true;
+ buildContrib = false;
+ doCheck = false;
+ };
+
+ cxxbridge = final.callPackage ./pkg/upstreamable/cxxbridge { };
+ wasmtime = final.callPackage ./pkg/upstreamable/wasmtime { };
+
+ scylla-driver = final.callPackage ./pkg/upstreamable/python-driver { };
+}
diff --git a/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix b/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix
--- a/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix
+++ b/dist/nix/pkg/custom/ccache-distcc-wrap/default.nix
@@ -0,0 +1,67 @@
+{ pkgs
+, lib
+
+, cc ? pkgs.cc
+, gcc ? pkgs.gcc
+, clang ? pkgs.clang
+
+, ... }:
+
+let
+ inherit (builtins)
+ map
+ match
+ ;
+ inherit (pkgs)
+ ccache
+ coreutils
+ distcc
+ runCommand
+ stdenv
+ ;
+
+ wrap = pkg: driver: ''
+ #! ${stdenv.shell} -e
+ driver=${pkg}/bin/${driver}
+ dist_driver="$driver${if ((match "clang.*" driver) != null) then " -Wno-error=unused-command-line-argument" else ""}"
+ distcc=${distcc}/bin/distcc
+ ccache=${ccache}/bin/ccache
+
+ export DISTCC_IO_TIMEOUT=1200 # hello, repair/row_level.cc
+
+ wrap=
+ if [[ -z "$NODISTCC" ]]; then
+ wrap=d
+ fi
+ if [[ -n "$CCACHE_DIR" ]]; then
+ wrap+=c
+ fi
+
+ if [[ -z "$wrap" ]]; then
+ exec $driver "$@"
+ elif [[ "$wrap" == d ]]; then
+ exec $distcc $dist_driver "$@"
+ elif [[ "$wrap" == c ]]; then
+ exec $ccache $driver "$@"
+ elif [[ "$wrap" == dc ]]; then
+ export CCACHE_PREFIX=$distcc
+ exec $ccache $dist_driver "$@"
+ else
+ echo wrapper bug 1>&2
+ exit 1
+ fi
+ '';
+in runCommand "distcc-ccache-wrap" { } ''
+ ${coreutils}/bin/mkdir -p $out/bin
+ ${lib.concatStrings (map ({pkg, driver}: ''
+ ${coreutils}/bin/echo ${lib.escapeShellArg (wrap pkg driver)} > $out/bin/${driver}
+ ${coreutils}/bin/chmod +x $out/bin/${driver}
+ '') [
+ { pkg = gcc; driver = "gcc"; }
+ { pkg = gcc; driver = "g++"; }
+ { pkg = clang; driver = "clang"; }
+ { pkg = clang; driver = "clang++"; }
+ { pkg = cc; driver = "cc"; }
+ { pkg = cc; driver = "c++"; }
+ ])}
+''
diff --git a/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock b/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock
--- a/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock
+++ b/dist/nix/pkg/upstreamable/cxxbridge/Cargo.lock
@@ -0,0 +1,447 @@
+# This file is automatically @generated by Cargo.
+# It is not intended for manual editing.
+version = 3
+
+[[package]]
+name = "adler"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f26201604c87b1e01bd3d98f8d5d9a8fcbb815e8cedb41ffccbeb4bf593a35fe"
+
+[[package]]
+name = "autocfg"
+version = "1.1.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d468802bab17cbc0cc575e9b053f41e72aa36bfa6b7f55e3529ffa43161b97fa"
+
+[[package]]
+name = "bitflags"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bef38d45163c2f1dde094a7dfd33ccf595c92905c8f8f4fdc18d06fb1037718a"
+
+[[package]]
+name = "cc"
+version = "1.0.73"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "2fff2a6927b3bb87f9595d67196a70493f627687a71d87a0d692242c33f58c11"
+dependencies = [
+ "jobserver",
+]
+
+[[package]]
+name = "cfg-if"
+version = "1.0.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "baf1de4339761588bc0619e3cbc0120ee582ebb74b53b4efbf79117bd2da40fd"
+
+[[package]]
+name = "clang-ast"
+version = "0.1.7"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d9c380e0de48337007dfc91b09eb75f567f5f08209437857fbaabbaf2e77e830"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "clap"
+version = "3.1.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2dbdf4bdacb33466e854ce889eee8dfd5729abf7ccd7664d0a2d60cd384440b"
+dependencies = [
+ "bitflags",
+ "clap_lex",
+ "indexmap",
+ "strsim",
+ "textwrap",
+]
+
+[[package]]
+name = "clap_lex"
+version = "0.2.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a37c35f1112dad5e6e0b1adaff798507497a18fceeb30cceb3bae7d1427b9213"
+dependencies = [
+ "os_str_bytes",
+]
+
+[[package]]
+name = "codespan-reporting"
+version = "0.11.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3538270d33cc669650c4b093848450d380def10c331d38c768e34cac80576e6e"
+dependencies = [
+ "termcolor",
+ "unicode-width",
+]
+
+[[package]]
+name = "crc32fast"
+version = "1.3.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b540bd8bc810d3885c6ea91e2018302f68baba2129ab3e88f32389ee9370880d"
+dependencies = [
+ "cfg-if",
+]
+
+[[package]]
+name = "cxx"
+version = "1.0.68"
+dependencies = [
+ "cc",
+ "cxx-build",
+ "cxx-gen",
+ "cxx-test-suite",
+ "cxxbridge-flags",
+ "cxxbridge-macro",
+ "link-cplusplus",
+ "rustversion",
+ "trybuild",
+]
+
+[[package]]
+name = "cxx-build"
+version = "1.0.68"
+dependencies = [
+ "cc",
+ "codespan-reporting",
+ "cxx-gen",
+ "once_cell",
+ "pkg-config",
+ "proc-macro2",
+ "quote",
+ "scratch",
+ "syn",
+]
+
+[[package]]
+name = "cxx-gen"
+version = "0.7.68"
+dependencies = [
+ "codespan-reporting",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "cxx-test-suite"
+version = "0.0.0"
+dependencies = [
+ "cxx",
+ "cxx-build",
+ "cxxbridge-flags",
+]
+
+[[package]]
+name = "cxxbridge-cmd"
+version = "1.0.68"
+dependencies = [
+ "clap",
+ "codespan-reporting",
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "cxxbridge-flags"
+version = "1.0.68"
+
+[[package]]
+name = "cxxbridge-macro"
+version = "1.0.68"
+dependencies = [
+ "clang-ast",
+ "cxx",
+ "flate2",
+ "memmap",
+ "proc-macro2",
+ "quote",
+ "serde",
+ "serde_json",
+ "syn",
+]
+
+[[package]]
+name = "demo"
+version = "0.0.0"
+dependencies = [
+ "cxx",
+ "cxx-build",
+]
+
+[[package]]
+name = "dissimilar"
+version = "1.0.4"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8c97b9233581d84b8e1e689cdd3a47b6f69770084fc246e86a7f78b0d9c1d4a5"
+
+[[package]]
+name = "flate2"
+version = "1.0.23"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b39522e96686d38f4bc984b9198e3a0613264abaebaff2c5c918bfa6b6da09af"
+dependencies = [
+ "cfg-if",
+ "crc32fast",
+ "libc",
+ "miniz_oxide",
+]
+
+[[package]]
+name = "glob"
+version = "0.3.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b919933a397b79c37e33b77bb2aa3dc8eb6e165ad809e58ff75bc7db2e34574"
+
+[[package]]
+name = "hashbrown"
+version = "0.11.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ab5ef0d4909ef3724cc8cce6ccc8572c5c817592e9285f5464f8e86f8bd3726e"
+
+[[package]]
+name = "indexmap"
+version = "1.8.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "0f647032dfaa1f8b6dc29bd3edb7bbef4861b8b8007ebb118d6db284fd59f6ee"
+dependencies = [
+ "autocfg",
+ "hashbrown",
+]
+
+[[package]]
+name = "itoa"
+version = "1.0.2"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "112c678d4050afce233f4f2852bb2eb519230b3cf12f33585275537d7e41578d"
+
+[[package]]
+name = "jobserver"
+version = "0.1.24"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "af25a77299a7f711a01975c35a6a424eb6862092cc2d6c72c4ed6cbc56dfc1fa"
+dependencies = [
+ "libc",
+]
+
+[[package]]
+name = "libc"
+version = "0.2.125"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5916d2ae698f6de9bfb891ad7a8d65c09d232dc58cc4ac433c7da3b2fd84bc2b"
+
+[[package]]
+name = "link-cplusplus"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f8cae2cd7ba2f3f63938b9c724475dfb7b9861b545a90324476324ed21dbc8c8"
+dependencies = [
+ "cc",
+]
+
+[[package]]
+name = "memmap"
+version = "0.7.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "6585fd95e7bb50d6cc31e20d4cf9afb4e2ba16c5846fc76793f11218da9c475b"
+dependencies = [
+ "libc",
+ "winapi",
+]
+
+[[package]]
+name = "miniz_oxide"
+version = "0.5.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "d2b29bd4bc3f33391105ebee3589c19197c4271e3e5a9ec9bfe8127eeff8f082"
+dependencies = [
+ "adler",
+]
+
+[[package]]
+name = "once_cell"
+version = "1.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "87f3e037eac156d1775da914196f0f37741a274155e34a0b7e427c35d2a2ecb9"
+
+[[package]]
+name = "os_str_bytes"
+version = "6.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "029d8d0b2f198229de29dca79676f2738ff952edf3fde542eb8bf94d8c21b435"
+
+[[package]]
+name = "pkg-config"
+version = "0.3.25"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1df8c4ec4b0627e53bdf214615ad287367e482558cf84b109250b37464dc03ae"
+
+[[package]]
+name = "proc-macro2"
+version = "1.0.38"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9027b48e9d4c9175fa2218adf3557f91c1137021739951d4932f5f8268ac48aa"
+dependencies = [
+ "unicode-xid",
+]
+
+[[package]]
+name = "quote"
+version = "1.0.18"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a1feb54ed693b93a84e14094943b84b7c4eae204c512b7ccb95ab0c66d278ad1"
+dependencies = [
+ "proc-macro2",
+]
+
+[[package]]
+name = "rustversion"
+version = "1.0.6"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f2cc38e8fa666e2de3c4aba7edeb5ffc5246c1c2ed0e3d17e560aeeba736b23f"
+
+[[package]]
+name = "ryu"
+version = "1.0.10"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "f3f6f92acf49d1b98f7a81226834412ada05458b7364277387724a237f062695"
+
+[[package]]
+name = "scratch"
+version = "1.0.1"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "96311ef4a16462c757bb6a39152c40f58f31cd2602a40fceb937e2bc34e6cbab"
+
+[[package]]
+name = "serde"
+version = "1.0.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "61ea8d54c77f8315140a05f4c7237403bf38b72704d031543aa1d16abbf517d1"
+dependencies = [
+ "serde_derive",
+]
+
+[[package]]
+name = "serde_derive"
+version = "1.0.137"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "1f26faba0c3959972377d3b2d306ee9f71faee9714294e41bb777f83f88578be"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "syn",
+]
+
+[[package]]
+name = "serde_json"
+version = "1.0.81"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "9b7ce2b32a1aed03c558dc61a5cd328f15aff2dbc17daad8fb8af04d2100e15c"
+dependencies = [
+ "itoa",
+ "ryu",
+ "serde",
+]
+
+[[package]]
+name = "strsim"
+version = "0.10.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "73473c0e59e6d5812c5dfe2a064a6444949f089e20eec9a2e5506596494e4623"
+
+[[package]]
+name = "syn"
+version = "1.0.94"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "a07e33e919ebcd69113d5be0e4d70c5707004ff45188910106854f38b960df4a"
+dependencies = [
+ "proc-macro2",
+ "quote",
+ "unicode-xid",
+]
+
+[[package]]
+name = "termcolor"
+version = "1.1.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "bab24d30b911b2376f3a13cc2cd443142f0c81dda04c118693e35b3835757755"
+dependencies = [
+ "winapi-util",
+]
+
+[[package]]
+name = "textwrap"
+version = "0.15.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "b1141d4d61095b28419e22cb0bbf02755f5e54e0526f97f1e3d1d160e60885fb"
+
+[[package]]
+name = "toml"
+version = "0.5.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "8d82e1a7758622a465f8cee077614c73484dac5b836c02ff6a40d5d1010324d7"
+dependencies = [
+ "serde",
+]
+
+[[package]]
+name = "trybuild"
+version = "1.0.61"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "7fc92f558afb6d1d7c6f175eb8d615b8ef49c227543e68e19c123d4ee43d8a7d"
+dependencies = [
+ "dissimilar",
+ "glob",
+ "once_cell",
+ "serde",
+ "serde_derive",
+ "serde_json",
+ "termcolor",
+ "toml",
+]
+
+[[package]]
+name = "unicode-width"
+version = "0.1.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "3ed742d4ea2bd1176e236172c8429aaf54486e7ac098db29ffe6529e0ce50973"
+
+[[package]]
+name = "unicode-xid"
+version = "0.2.3"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "957e51f3646910546462e67d5f7599b9e4fb8acdd304b087a6494730f9eebf04"
+
+[[package]]
+name = "winapi"
+version = "0.3.9"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "5c839a674fcd7a98952e593242ea400abe93992746761e38641405d28b00f419"
+dependencies = [
+ "winapi-i686-pc-windows-gnu",
+ "winapi-x86_64-pc-windows-gnu",
+]
+
+[[package]]
+name = "winapi-i686-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "ac3b87c63620426dd9b991e5ce0329eff545bccbbb34f3be09ff6fb6ab51b7b6"
+
+[[package]]
+name = "winapi-util"
+version = "0.1.5"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "70ec6ce85bb158151cae5e5c87f95a8e97d2c0c4b001223f33a334e3ce5de178"
+dependencies = [
+ "winapi",
+]
+
+[[package]]
+name = "winapi-x86_64-pc-windows-gnu"
+version = "0.4.0"
+source = "registry+https://github.com/rust-lang/crates.io-index"
+checksum = "712e227841d057c1ee1cd2fb22fa7e5a5461ae8e48fa2ca79ec42cfc1931183f"
diff --git a/dist/nix/pkg/upstreamable/cxxbridge/default.nix b/dist/nix/pkg/upstreamable/cxxbridge/default.nix
--- a/dist/nix/pkg/upstreamable/cxxbridge/default.nix
+++ b/dist/nix/pkg/upstreamable/cxxbridge/default.nix
@@ -0,0 +1,28 @@
+{ pkgs
+}:
+
+let
+ inherit (pkgs)
+ fetchFromGitHub
+ rustPlatform
+ ;
+
+ version = "1.0.68";
+ lockFile = ./Cargo.lock;
+
+in rustPlatform.buildRustPackage {
+ pname = "cxxbridge-cmd";
+ inherit version;
+
+ src = fetchFromGitHub {
+ owner = "dtolnay";
+ repo = "cxx";
+ rev = "${version}";
+ sha256 = "sha256-DdcbPcxTGJ5rJUJxQR3YBHbe9g3JjgbP/htJqWerRlI=";
+ };
+
+ cargoLock = { inherit lockFile; };
+ postPatch = "cp ${lockFile} Cargo.lock";
+
+ cargoBuildFlags = [ "--package cxxbridge-cmd" ];
+}
diff --git a/dist/nix/pkg/upstreamable/python-driver/default.nix b/dist/nix/pkg/upstreamable/python-driver/default.nix
--- a/dist/nix/pkg/upstreamable/python-driver/default.nix
+++ b/dist/nix/pkg/upstreamable/python-driver/default.nix
@@ -0,0 +1,31 @@
+{ stdenv
+, lib
+, fetchFromGitHub
+, python3Packages
+, libev
+}:
+
+let
+ version = "3.25.4-scylla";
+in python3Packages.buildPythonPackage {
+ pname = "scylla-driver";
+ inherit version;
+
+ # pypi tarball doesn't include tests
+ src = fetchFromGitHub {
+ owner = "scylladb";
+ repo = "python-driver";
+ rev = version;
+ sha256 = "sha256-LIPZ4sY/wrhmy+kpFUwBvgvbJoXanQLkzMd5Iv0X2mc=";
+ };
+
+ postPatch = ''
+ substituteInPlace setup.py --replace 'geomet>=0.1,<0.3' 'geomet'
+ '';
+
+ nativeBuildInputs = with python3Packages; [ cython ];
+ buildInputs = [ libev ];
+ propagatedBuildInputs = with python3Packages; [ six geomet ];
+
+ doCheck = false;
+}
diff --git a/dist/nix/pkg/upstreamable/wasmtime/default.nix b/dist/nix/pkg/upstreamable/wasmtime/default.nix
--- a/dist/nix/pkg/upstreamable/wasmtime/default.nix
+++ b/dist/nix/pkg/upstreamable/wasmtime/default.nix
@@ -0,0 +1,47 @@
+{ pkgs
+}:
+
+let
+ inherit (pkgs)
+ cmake
+ fetchFromGitHub
+ python3
+ rustPlatform
+ ;
+
+ llvm = pkgs.llvmPackages_latest;
+ clang = llvm.clang;
+
+ pname = "wasmtime";
+ version = "0.29.0";
+
+in rustPlatform.buildRustPackage {
+ inherit pname version;
+ src = fetchFromGitHub {
+ owner = "bytecodealliance";
+ repo = pname;
+ rev = "v${version}";
+ sha256 = "sha256-qmME9zI2vSFQlJWhve7dqAPh8O/6WclWTwO9Pcdvoj8=";
+ fetchSubmodules = true;
+ };
+
+ cargoSha256 = "sha256-xtx1iCtZ9rRJec6Q7ywBM3ov9z0YJMcxz7K59Ox6DKk=";
+
+ nativeBuildInputs = [ python3 cmake clang ];
+ buildInputs = [ llvm.libclang ];
+ LIBCLANG_PATH = "${llvm.libclang.lib}/lib";
+
+ cargoBuildFlags = [ "--package wasmtime-c-api" ];
+
+ # cargo does not install the C(++) headers
+ postInstall = ''
+ install -d -m744 $out/include/wasmtime
+ install -m644 $src/crates/c-api/include/*.h $out/include
+ install -m644 $src/crates/c-api/include/wasmtime/*.h $out/include/wasmtime
+ install -m644 $src/crates/c-api/wasm-c-api/include/* $out/include
+ '';
+
+ # Scylla has to use this version (later ones change APIs), and it
+ # happens not to pass its own tests, so:
+ doCheck = false;
+}
diff --git a/flake.lock b/flake.lock
--- a/flake.lock
+++ b/flake.lock
@@ -0,0 +1,43 @@
+{
+ "nodes": {
+ "flake-utils": {
+ "locked": {
+ "lastModified": 1656928814,
+ "narHash": "sha256-RIFfgBuKz6Hp89yRr7+NR5tzIAbn52h8vT6vXkYjZoM=",
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "rev": "7e2a3b3dfd9af950a856d66b0a7d01e3c18aa249",
+ "type": "github"
+ },
+ "original": {
+ "owner": "numtide",
+ "repo": "flake-utils",
+ "type": "github"
+ }
+ },
+ "nixpkgs": {
+ "locked": {
+ "lastModified": 1657532632,
+ "narHash": "sha256-uEwe1CAQxb0eOWIwxQ3zpMs/llshirlrCXZQz6vkYQU=",
+ "owner": "NixOS",
+ "repo": "nixpkgs",
+ "rev": "85deee6d6c8127d360096a5caa0aeb876b976496",
+ "type": "github"
+ },
+ "original": {
+ "owner": "NixOS",
+ "ref": "nixpkgs-unstable",
+ "repo": "nixpkgs",
+ "type": "github"
+ }
+ },
+ "root": {
+ "inputs": {
+ "flake-utils": "flake-utils",
+ "nixpkgs": "nixpkgs"
+ }
+ }
+ },
+ "root": "root",
+ "version": 7
+}
diff --git a/flake.nix b/flake.nix
--- a/flake.nix
+++ b/flake.nix
@@ -0,0 +1,51 @@
+{
+ description = "Monstrously Fast + Scalable NoSQL";
+
+ inputs = {
+ nixpkgs.url = "github:NixOS/nixpkgs/nixpkgs-unstable";
+ flake-utils.url = "github:numtide/flake-utils";
+ };
+
+ outputs = { self, nixpkgs, flake-utils } @ inputs: {
+ overlays.default = import ./dist/nix/overlay.nix nixpkgs;
+
+ lib = {
+ _attrs = system: let
+ pkgs = import nixpkgs {
+ inherit system;
+ overlays = [ self.overlays.default ];
+ };
+
+ repl = pkgs.writeText "repl" ''
+ let
+ self = builtins.getFlake (toString ${self.outPath});
+ attrs = self.lib._attrs "${system}";
+ in {
+ inherit self;
+ inherit (attrs) pkgs;
+ }
+ '';
+
+ args = {
+ flake = true;
+ srcPath = "${self}";
+ inherit pkgs repl;
+ };
+
+ package = import ./default.nixpkgs args;
+ devShell = import ./shell.nix args;
+ in {
+ inherit pkgs args package devShell;
+ };
+ };
+ }
+ // (flake-utils.lib.eachDefaultSystem (system: let
+ packageName = "scylla";
+ attrs = self.lib._attrs system;
+ in {
+ packages.${packageName} = attrs.package;
+ defaultPackage = self.packages.${system}.${packageName};
+
+ inherit (attrs) devShell;
+ }));
+}
diff --git a/shell.nix b/shell.nix
--- a/shell.nix
+++ b/shell.nix
@@ -4,18 +4,27 @@
#
# SPDX-License-Identifier: AGPL-3.0-or-later

-{
- pkgs ? null,
- mode ? "dev",
- useCcache ? false
-}:
-import ./default.nix ({
- inherit mode useCcache;
- testInputsFrom = pkgs: with pkgs; [
- python3Packages.boto3
- python3Packages.colorama
- python3Packages.pytest
+args:
+
+import ./default.nix (args // {
+ shell = true;
+
+ devInputs = { pkgs, llvm }: with pkgs; [
+ # for impure building
+ ccache
+ distcc
+
+ # for debugging
+ binutils # addr2line etc.
+ elfutils
+
+ gdbWithGreenThreadSupport
+
+ llvm.llvm
+ lz4 # coredumps on modern Systemd installations are lz4-compressed
+
+ # etc
+ diffutils
+ colordiff
];
- gitPkg = pkgs: pkgs.gitFull;
-} //
-(if pkgs != null then { inherit pkgs; } else {}))
+})

[Commit Bot]

From: Botond Dénes <bde...@scylladb.com>
Committer: Botond Dénes <bde...@scylladb.com>

Branch: master