diff --git a/Makefile b/Makefile index 231bbe11f4c011..a85cf7b46d9918 100644 --- a/Makefile +++ b/Makefile @@ -1212,6 +1212,7 @@ $(TARBALL): release-only doc-only mkdir -p $(TARNAME)/doc/api cp doc/node.1 $(TARNAME)/doc/node.1 cp -r out/doc/api/* $(TARNAME)/doc/api/ + sed 's/fileset = fileset.intersection (fileset.gitTracked root)/fileset =/' tools/nix/v8.nix > $(TARNAME)/tools/nix/v8.nix $(RM) -r $(TARNAME)/.editorconfig $(RM) -r $(TARNAME)/.git* $(RM) -r $(TARNAME)/.mailmap diff --git a/shell.nix b/shell.nix index fc20fca2ce095e..382d112534f94c 100644 --- a/shell.nix +++ b/shell.nix @@ -18,21 +18,18 @@ withSQLite ? true, withSSL ? true, withTemporal ? false, - sharedLibDeps ? - let - d = import ./tools/nix/sharedLibDeps.nix { - inherit - pkgs - withLief - withQuic - withSQLite - withSSL - withTemporal - ; - }; - in - # To avoid conflicts with V8's bundled simdutf lib, it's easier to remove it when using a precompiled V8. - if (useSeparateDerivationForV8 != false) then builtins.removeAttrs d [ "simdutf" ] else d, + sharedLibDeps ? ( + import ./tools/nix/sharedLibDeps.nix { + inherit + pkgs + withLief + withQuic + withSQLite + withSSL + withTemporal + ; + } + ), # dev tools (not needed to build Node.js, useful to maintain it) ncu-path ? null, # Provide this if you want to use a local version of NCU @@ -45,9 +42,20 @@ let useSharedAda = builtins.hasAttr "ada" sharedLibDeps; useSharedOpenSSL = builtins.hasAttr "openssl" sharedLibDeps; - needsRustCompiler = withTemporal && !builtins.hasAttr "temporal_capi" sharedLibDeps; + useSharedTemporal = builtins.hasAttr "temporal_capi" sharedLibDeps; + needsRustCompiler = withTemporal && !useSharedTemporal; - buildInputs = builtins.attrValues sharedLibDeps ++ pkgs.lib.optional useSharedICU icu; + nativeBuildInputs = + pkgs.nodejs-slim_latest.nativeBuildInputs + ++ pkgs.lib.optionals needsRustCompiler [ + pkgs.cargo + pkgs.rustc + ]; + buildInputs = + pkgs.lib.optional useSharedICU icu + ++ pkgs.lib.optional (withTemporal && useSharedTemporal) sharedLibDeps.temporal_capi; + + # Put here only the configure flags that affect the V8 build configureFlags = [ ( if icu == null then @@ -56,48 +64,31 @@ let "--with-intl=${if useSharedICU then "system" else icu}-icu" ) ] - ++ extraConfigFlags - ++ pkgs.lib.optional (!withAmaro) "--without-amaro" - ++ pkgs.lib.optional (!withLief) "--without-lief" - ++ pkgs.lib.optional withQuic "--experimental-quic" - ++ pkgs.lib.optional (!withSQLite) "--without-sqlite" - ++ pkgs.lib.optional (!withSSL) "--without-ssl" ++ pkgs.lib.optional withTemporal "--v8-enable-temporal-support" - ++ pkgs.lib.optional (ninja != null) "--ninja" - ++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ./.}" - ++ pkgs.lib.concatMap (name: [ - "--shared-${name}" - "--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib" - "--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include" - ]) (builtins.attrNames sharedLibDeps); + ++ pkgs.lib.optional (withTemporal && useSharedTemporal) "--shared-temporal_capi"; in pkgs.mkShell { - inherit (pkgs.nodejs-slim_latest) nativeBuildInputs; + inherit nativeBuildInputs; buildInputs = - buildInputs + builtins.attrValues sharedLibDeps + ++ buildInputs ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) ( if useSeparateDerivationForV8 == true then - import ./tools/nix/v8.nix { - inherit - pkgs - configureFlags - buildInputs - needsRustCompiler - ; + let + sharedLibsToMock = pkgs.callPackage ./tools/nix/non-v8-deps-mock.nix { }; + in + pkgs.callPackage ./tools/nix/v8.nix { + inherit nativeBuildInputs icu; + + configureFlags = configureFlags ++ sharedLibsToMock.configureFlags ++ [ "--ninja" ]; + buildInputs = buildInputs ++ [ sharedLibsToMock ]; } else useSeparateDerivationForV8 ); - packages = - pkgs.lib.optional (ccache != null) ccache - ++ devTools - ++ benchmarkTools - ++ pkgs.lib.optionals needsRustCompiler [ - pkgs.cargo - pkgs.rustc - ]; + packages = devTools ++ benchmarkTools ++ pkgs.lib.optional (ccache != null) ccache; shellHook = pkgs.lib.optionalString (ccache != null) '' export CC="${pkgs.lib.getExe ccache} $CC" @@ -107,7 +98,34 @@ pkgs.mkShell { BUILD_WITH = if (ninja != null) then "ninja" else "make"; NINJA = pkgs.lib.optionalString (ninja != null) "${pkgs.lib.getExe ninja}"; CONFIG_FLAGS = builtins.toString ( - configureFlags ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) "--without-bundled-v8" + configureFlags + ++ extraConfigFlags + ++ pkgs.lib.optional (ninja != null) "--ninja" + ++ pkgs.lib.optional (!withAmaro) "--without-amaro" + ++ pkgs.lib.optional (!withLief) "--without-lief" + ++ pkgs.lib.optional withQuic "--experimental-quic" + ++ pkgs.lib.optional (!withSQLite) "--without-sqlite" + ++ pkgs.lib.optional (!withSSL) "--without-ssl" + ++ pkgs.lib.optional loadJSBuiltinsDynamically "--node-builtin-modules-path=${builtins.toString ./.}" + ++ pkgs.lib.optional (useSeparateDerivationForV8 != false) "--without-bundled-v8" + ++ + pkgs.lib.concatMap + (name: [ + "--shared-${name}" + "--shared-${name}-libpath=${pkgs.lib.getLib sharedLibDeps.${name}}/lib" + "--shared-${name}-include=${pkgs.lib.getInclude sharedLibDeps.${name}}/include" + ]) + ( + builtins.attrNames ( + if (useSeparateDerivationForV8 != false) then + builtins.removeAttrs sharedLibDeps [ + "simdutf" + "temporal_capi" + ] + else + sharedLibDeps + ) + ) ); NOSQLITE = pkgs.lib.optionalString (!withSQLite) "1"; } diff --git a/tools/nix/non-v8-deps-mock.nix b/tools/nix/non-v8-deps-mock.nix new file mode 100644 index 00000000000000..46933fd09c282b --- /dev/null +++ b/tools/nix/non-v8-deps-mock.nix @@ -0,0 +1,76 @@ +{ + symlinkJoin, + writeTextFile, + validatePkgConfig, + testers, + lib, +}: + +let + sharedLibsToMock = { + zlib = [ "zlib" ]; + http-parser = [ "libllhttp" ]; + libuv = [ "libuv" ]; + ada = [ "ada" ]; + simdjson = [ "simdjson" ]; + brotli = [ + "libbrotlidec" + "libbrotlienc" + ]; + cares = [ "libcares" ]; + gtest = [ "gtest" ]; + hdr-histogram = [ "hdr_histogram" ]; + merve = [ "merve" ]; + nbytes = [ "nbytes" ]; + nghttp2 = [ "libnghttp2" ]; + nghttp3 = [ "libnghttp3" ]; + ngtcp2 = [ "libngtcp2" ]; + uvwasi = [ "uvwasi" ]; + zstd = [ "libzstd" ]; + }; +in +symlinkJoin (finalAttrs: { + pname = "non-v8-deps-mock"; + version = "0.0.0-mock"; + + nativeBuildInputs = [ validatePkgConfig ]; + + paths = lib.concatMap ( + sharedLibName: + (builtins.map ( + pkgName: + writeTextFile { + name = "mock-${pkgName}.pc"; + destination = "/lib/pkgconfig/${pkgName}.pc"; + text = '' + Name: ${pkgName} + Description: Mock package for ${sharedLibName} + Version: ${finalAttrs.version} + Libs: + Cflags: + ''; + } + ) sharedLibsToMock.${sharedLibName}) + ) (builtins.attrNames sharedLibsToMock); + passthru = { + configureFlags = [ + "--without-lief" + "--without-sqlite" + "--without-ssl" + ] + ++ (lib.concatMap (sharedLibName: [ + "--shared-${sharedLibName}" + "--shared-${sharedLibName}-libname=" + ]) (builtins.attrNames sharedLibsToMock)); + + tests.pkg-config = testers.hasPkgConfigModules { + package = finalAttrs.finalPackage; + }; + }; + + meta = { + description = "Mock of Node.js dependencies that are not needed for building V8"; + license = lib.licenses.mit; + pkgConfigModules = lib.concatMap (x: x) (builtins.attrValues sharedLibsToMock); + }; +}) diff --git a/tools/nix/temporal-no-vendored-icu.patch b/tools/nix/temporal-no-vendored-icu.patch new file mode 100644 index 00000000000000..d9a06b3ce6945f --- /dev/null +++ b/tools/nix/temporal-no-vendored-icu.patch @@ -0,0 +1,63 @@ +diff --git a/deps/v8/src/objects/js-temporal-zoneinfo64.cc b/deps/v8/src/objects/js-temporal-zoneinfo64.cc +index 99dd3a84c1e..b6b399c10dc 100644 +--- a/deps/v8/src/objects/js-temporal-zoneinfo64.cc ++++ b/deps/v8/src/objects/js-temporal-zoneinfo64.cc +@@ -11,12 +11,44 @@ + #include "temporal_rs/TimeZone.hpp" + + #ifdef V8_INTL_SUPPORT +-#include "udatamem.h" ++#include "unicode/udata.h" ++typedef struct { ++ uint16_t headerSize; ++ uint8_t magic1; ++ uint8_t magic2; ++} MappedData; ++typedef struct { ++ MappedData dataHeader; ++ UDataInfo info; ++} DataHeader; ++typedef struct { ++ void* Lookup; ++ void* NumEntries; ++} commonDataFuncs; ++struct UDataMemory { ++ const commonDataFuncs *vFuncs; /* Function Pointers for accessing TOC */ ++ ++ const DataHeader *pHeader; /* Header of the memory being described by this */ ++ /* UDataMemory object. */ ++ const void *toc; /* For common memory, table of contents for */ ++ /* the pieces within. */ ++ UBool heapAllocated; /* True if this UDataMemory Object is on the */ ++ /* heap and thus needs to be deleted when closed. */ ++ ++ void *mapAddr; /* For mapped or allocated memory, the start addr. */ ++ /* Only non-null if a close operation should unmap */ ++ /* the associated data. */ ++ void *map; /* Handle, or other data, OS dependent. */ ++ /* Only non-null if a close operation should unmap */ ++ /* the associated data, and additional info */ ++ /* beyond the mapAddr is needed to do that. */ ++ int32_t length; /* Length of the data in bytes; -1 if unknown. */ ++}; + #else + // Defined in builtins-temporal-zoneinfo64-data.cc, generated by + // include-file-as-bytes.py +-extern "C" uint32_t zoneinfo64_static_data[]; +-extern "C" size_t zoneinfo64_static_data_len; ++static uint32_t zoneinfo64_static_data[] = {}; ++static size_t zoneinfo64_static_data_len = 0; + #endif + + namespace v8::internal { +@@ -33,7 +65,7 @@ ZoneInfo64Provider::ZoneInfo64Provider() { + // NOT udata_getLength: this ignores the header, + // and we're parsing resb files with the header + auto length = memory->length; +- const void* data = udata_getRawMemory(memory); ++ const void* data = udata_getMemory(memory); + DCHECK_WITH_MSG(length % 4 == 0, "ICU4C should align udata to uint32_t"); + if (length % 4 != 0) { + // This really shouldn't happen: ICU4C aligns these files +-- +2.51.0 diff --git a/tools/nix/v8.nix b/tools/nix/v8.nix index bb52db491ecaa4..73356f067c3a63 100644 --- a/tools/nix/v8.nix +++ b/tools/nix/v8.nix @@ -1,96 +1,142 @@ # Derivation for Node.js CI (not officially supported for regular applications) { - pkgs ? import ./pkgs.nix { }, - buildInputs ? [ ], - configureFlags ? [ ], - needsRustCompiler ? false, + stdenv, + lib, + patchutils, + validatePkgConfig, + icu, + + buildInputs ? lib.optional (icu != null) icu, + configureFlags ? [ + ( + if icu == null then + "--without-intl" + else + "--with-intl=${if builtins.isString icu then icu else "system"}-icu" + ) + ], + + nodejs-slim_latest ? null, + configureScript ? nodejs-slim_latest.configureScript, + nativeBuildInputs ? nodejs-slim_latest.nativeBuildInputs, + patches ? nodejs-slim_latest.patches, }: let - nodejs = pkgs.nodejs-slim_latest; - v8Dir = ../../deps/v8; - - version = - let - v8Version = builtins.match ( - ".*#define V8_MAJOR_VERSION ([0-9]+).*" - + "#define V8_MINOR_VERSION ([0-9]+).*" - + "#define V8_BUILD_NUMBER ([0-9]+).*" - + "#define V8_PATCH_LEVEL ([0-9]+).*" - ) (builtins.readFile "${v8Dir}/include/v8-version.h"); - v8_embedder_string = builtins.match ".*'v8_embedder_string': '-(node.[0-9]+)'.*" ( - builtins.readFile ../../common.gypi - ); - in - if v8Version == null || v8_embedder_string == null then - throw "V8 version not found" - else - "${builtins.elemAt v8Version 0}.${builtins.elemAt v8Version 1}.${builtins.elemAt v8Version 2}.${builtins.elemAt v8Version 3}-${builtins.elemAt v8_embedder_string 0}"; -in -pkgs.stdenv.mkDerivation (finalAttrs: { - pname = "v8"; - inherit version; src = let - inherit (pkgs.lib) fileset; - in - fileset.toSource { + inherit (lib) fileset; root = ../../.; - fileset = fileset.unions [ + files = [ ../../common.gypi - ../../configure ../../configure.py - ../../deps/inspector_protocol/inspector_protocol.gyp - ../../deps/ncrypto/ncrypto.gyp - v8Dir + ../../deps/v8 ../../node.gyp ../../node.gypi - ../../src/inspector/node_inspector.gypi ../../src/node_version.h - ../../tools/configure.d + ../../tools/configure.d/nodedownload.py ../../tools/getmoduleversion.py ../../tools/getnapibuildversion.py - ../../tools/gyp + ../../tools/gyp/pylib ../../tools/gyp_node.py + ../../tools/utils.py + ../../tools/v8_gypfiles/abseil.gyp + ../../tools/v8_gypfiles/features.gypi + ../../tools/v8_gypfiles/ForEachFormat.py + ../../tools/v8_gypfiles/ForEachReplace.py + ../../tools/v8_gypfiles/GN-scraper.py + ../../tools/v8_gypfiles/inspector.gypi + ../../tools/v8_gypfiles/toolchain.gypi + ../../tools/v8_gypfiles/v8.gyp + ] + ++ lib.optionals (icu != null) [ ../../tools/icu/icu_versions.json ../../tools/icu/icu-system.gyp - ../../tools/utils.py - ../../tools/v8_gypfiles + ] + ++ lib.optionals (icu == "small") [ + ../../deps/icu-small + ../../tools/icu/current_ver.dep + ../../tools/icu/icu_small.json + ../../tools/icu/icu-generic.gyp + ../../tools/icu/iculslocs.cc + ../../tools/icu/icutrim.py + ../../tools/icu/no-op.cc ]; + in + fileset.toSource { + inherit root; + fileset = fileset.intersection (fileset.gitTracked root) (fileset.unions files); }; + v8Dir = "${src}/deps/v8"; +in +stdenv.mkDerivation (finalAttrs: { + pname = "v8"; + version = + let + v8Version = builtins.match ( + ".*#define V8_MAJOR_VERSION ([0-9]+).*" + + "#define V8_MINOR_VERSION ([0-9]+).*" + + "#define V8_BUILD_NUMBER ([0-9]+).*" + + "#define V8_PATCH_LEVEL ([0-9]+).*" + ) (builtins.readFile "${v8Dir}/include/v8-version.h"); + v8_embedder_string = builtins.match ".*'v8_embedder_string': '-(node.[0-9]+)'.*" ( + builtins.readFile "${src}/common.gypi" + ); + in + if v8Version == null || v8_embedder_string == null then + throw "V8 version not found" + else + "${builtins.elemAt v8Version 0}.${builtins.elemAt v8Version 1}.${builtins.elemAt v8Version 2}.${builtins.elemAt v8Version 3}-${builtins.elemAt v8_embedder_string 0}"; + + patches = lib.optional ( + # V8 accesses internal ICU headers and methods in the Temporal files. + !(builtins.isString icu) && builtins.elem "--v8-enable-temporal-support" configureFlags + ) ./temporal-no-vendored-icu.patch; # We need to patch tools/gyp/ to work from within Nix sandbox prePatch = '' - patches=() - for patch in ${pkgs.lib.concatStringsSep " " nodejs.patches}; do + ${lib.optionalString (builtins.length finalAttrs.patches == 0) "patches=()"} + for patch in ${lib.concatStringsSep " " patches}; do filtered=$(mktemp) - filterdiff -p1 -i 'tools/gyp/*' "$patch" > "$filtered" + filterdiff -p1 -i 'tools/gyp/pylib/*' "$patch" > "$filtered" if [ -s "$filtered" ]; then patches+=("$filtered") fi done ''; + # We need to remove the node_inspector.gypi ref so GYP does not search for it. + postPatch = '' + substituteInPlace node.gyp --replace-fail "'includes' : [ 'src/inspector/node_inspector.gypi' ]" "'includes' : []" + '' + + lib.optionalString (icu == null) '' + substituteInPlace configure.py \ + --replace-fail \ + "icu_versions = json.loads((tools_path / 'icu' / 'icu_versions.json').read_text(encoding='utf-8'))" \ + "icu_versions = { 'minimum_icu': 1 }" + ''; - inherit (nodejs) configureScript; - inherit configureFlags buildInputs; + inherit + src + configureScript + configureFlags + buildInputs + ; - nativeBuildInputs = - nodejs.nativeBuildInputs - ++ [ - pkgs.patchutils - pkgs.validatePkgConfig - ] - ++ pkgs.lib.optionals needsRustCompiler [ - pkgs.cargo - pkgs.rustc - ]; + nativeBuildInputs = nativeBuildInputs ++ [ + patchutils + validatePkgConfig + ]; buildPhase = '' + runHook preBuild ninja -v -C out/Release v8_snapshot v8_libplatform + runHook postBuild ''; installPhase = '' + runHook preInstall + ${ - if pkgs.stdenv.hostPlatform.isDarwin then + if stdenv.hostPlatform.isDarwin then # Darwin is excluded from creating thin archive in tools/gyp/pylib/gyp/generator/ninja.py:2488 "install -Dm644 out/Release/lib* -t $out/lib" else @@ -111,16 +157,31 @@ pkgs.stdenv.mkDerivation (finalAttrs: { '' } + install -Dm644 deps/v8/third_party/simdutf/simdutf.h -t $out/include + find deps/v8/include -name '*.h' -print0 | while read -r -d "" file; do + install -Dm644 "$file" -T "$out/include/''${file#deps/v8/include/}" + done + find deps/v8/third_party/abseil-cpp/absl -name '*.h' -print0 | while read -r -d "" file; do + install -Dm644 "$file" -T "$out/include/''${file#deps/v8/third_party/abseil-cpp/}" + done + mkdir -p $out/lib/pkgconfig cat -> $out/lib/pkgconfig/v8.pc << EOF + prefix=$out + exec_prefix=\''${prefix} + libdir=\''${exec_prefix}/lib + includedir=\''${prefix}/include + Name: v8 Description: V8 JavaScript Engine build for Node.js CI - Version: ${version} - Libs: -L$out/lib $(for f in $out/lib/lib*.a; do + Version: ${finalAttrs.version} + Libs: -L\''${libdir} $(for f in $out/lib/lib*.a; do b=$(basename "$f" .a) printf " -l%s" "''${b#lib}" done) -lstdc++ - Cflags: -I${v8Dir}/include -I${v8Dir}/third_party/abseil-cpp -I${v8Dir}/third_party/simdutf + Cflags: -I\''${includedir} EOF + + runHook postInstall ''; })