From cb9e7332f4f9142d6557f74fd67df81af9fdc8e7 Mon Sep 17 00:00:00 2001 From: Steffen Beyer Date: Mon, 6 Apr 2026 12:38:35 +0200 Subject: [PATCH] system: separate build profile from development --- docs/GUIX_DIFFERENCES.md | 40 ++-- docs/PROGRESS.md | 55 +++++ ...hase20-build-profile-separation-freebsd.md | 211 ++++++++++++++++++ docs/system-deployment-workflow.md | 69 ++++-- modules/fruix/system/freebsd.scm | 1 + modules/fruix/system/freebsd/build.scm | 10 +- modules/fruix/system/freebsd/media.scm | 37 ++- modules/fruix/system/freebsd/model.scm | 21 +- modules/fruix/system/freebsd/render.scm | 75 +++++-- ...hase20-development-operating-system.scm.in | 2 + ...n-phase20-development-environment-xcpng.sh | 42 +++- ...ase20-host-initiated-native-build-xcpng.sh | 7 +- ...se20-native-build-store-promotion-xcpng.sh | 6 +- ...-phase20-self-hosted-native-build-xcpng.sh | 9 +- 14 files changed, 517 insertions(+), 68 deletions(-) create mode 100644 docs/reports/postphase20-build-profile-separation-freebsd.md diff --git a/docs/GUIX_DIFFERENCES.md b/docs/GUIX_DIFFERENCES.md index 032bee1..95fab5e 100644 --- a/docs/GUIX_DIFFERENCES.md +++ b/docs/GUIX_DIFFERENCES.md @@ -248,37 +248,45 @@ For Guix-familiar operators, the practical takeaway is: - do **not** assume Fruix store prefixes are byte-for-byte comparable to Guix/Nix ones - expect Fruix to prefer a simpler, centralized naming policy unless exact Guix/Nix behavior becomes necessary later -## 8. Fruix can expose a separate in-system development profile overlay +## 8. Fruix can expose separate in-system development and build overlays -For the validated Phase 20.1 path, Fruix can now expose development tooling separately from the main runtime profile. +For the validated Phase 20 path, Fruix now distinguishes three layers instead of two: + +- runtime profile +- development profile +- build profile On those systems, Fruix exposes: -- `/run/current-system/development-profile` -- `/run/current-development` -- `/usr/local/bin/fruix-development-environment` -- `/usr/include -> /run/current-system/development-profile/usr/include` -- `/usr/share/mk -> /run/current-system/development-profile/usr/share/mk` +- development: + - `/run/current-system/development-profile` + - `/run/current-development` + - `/usr/local/bin/fruix-development-environment` +- build: + - `/run/current-system/build-profile` + - `/run/current-build` + - `/usr/local/bin/fruix-build-environment` +- canonical base-build compatibility links: + - `/usr/include -> /run/current-system/build-profile/usr/include` + - `/usr/share/mk -> /run/current-system/build-profile/usr/share/mk` The intent is: - keep the main runtime profile lean -- expose headers, `usr/share/mk`, and selected toolchain commands explicitly -- satisfy native FreeBSD buildworld/buildkernel expectations for canonical system paths when development support is enabled -- avoid treating a development-heavy system image as the default runtime shape +- keep the development helper interactive and convenient +- keep the build helper narrower, more sanitized, and closer to the actual `buildworld` / `buildkernel` contract +- avoid treating a development-heavy or build-heavy image as the default runtime shape -Compared with Guix, this is conceptually similar to keeping development-oriented state separate from the main runtime identity, but Fruix currently expresses it as a system-attached development overlay rather than through Guix's broader profile/tooling model. +Compared with Guix, this is conceptually similar to keeping development-oriented and build-oriented state separate from the main runtime identity, but Fruix currently expresses it as system-attached overlays rather than through Guix's broader profile/tooling model. -Fruix now also has a narrow guest self-hosted native-build prototype helper at: +Fruix still has the guest self-hosted native-build prototype helper at: - `/usr/local/bin/fruix-self-hosted-native-build` -That helper does **not** just reuse the whole exported development shell wholesale. The validated prototype had to sanitize development-oriented variables such as `MAKEFLAGS`, `CPPFLAGS`, `CFLAGS`, `CXXFLAGS`, and `LDFLAGS` before `buildworld`, because those are convenient for smaller development tasks but can poison the FreeBSD world/kernel bootstrap path. - -The practical Fruix takeaway is: +But that helper now explicitly uses the build helper contract instead of trying to reuse the full interactive development shell. The practical Fruix takeaway is: - the development overlay makes native base work possible inside the system -- but real guest self-hosted base builds still need their own stricter build contract +- the build overlay gives Fruix a stricter, more reproducible contract for real base builds Fruix now also makes an explicit distinction between: diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index b57fd05..e3eae95 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -38,6 +38,12 @@ Fruix currently has: - `/run/current-system/development-profile` - `/run/current-development` - `/usr/local/bin/fruix-development-environment` +- a validated separate in-system build environment overlay via: + - `/run/current-system/build-profile` + - `/run/current-build` + - `/usr/local/bin/fruix-build-environment` + - `/usr/include -> /run/current-system/build-profile/usr/include` + - `/usr/share/mk -> /run/current-system/build-profile/usr/share/mk` - a validated host-initiated native base-build path inside a Fruix-managed guest via: - real XCP-ng boot of a development-enabled Fruix system - in-guest `buildworld` / `buildkernel` @@ -81,6 +87,55 @@ The validated Phase 18 installation work currently uses: ## Latest completed achievement +### 2026-04-06 — Fruix now separates interactive development from strict native base-build environment + +Fruix now has a more explicit three-layer model for build-capable FreeBSD systems: + +- runtime profile +- development profile +- build profile + +Highlights: + +- `` now supports separate `build-packages` +- system closures can now materialize both: + - `development-profile` + - `build-profile` +- build-capable systems now expose: + - `/run/current-system/build-profile` + - `/run/current-build` + - `/usr/local/bin/fruix-build-environment` +- canonical compatibility links for native base builds now come from the build profile: + - `/usr/include -> /run/current-system/build-profile/usr/include` + - `/usr/share/mk -> /run/current-system/build-profile/usr/share/mk` +- the new build helper intentionally clears development-shell variables such as: + - `MAKEFLAGS` + - `CPPFLAGS` + - `CFLAGS` + - `CXXFLAGS` + - `LDFLAGS` +- the self-hosted native-build helper now uses this stricter build-helper contract instead of manually reconstructing that sanitization ad hoc +- promotion metadata for native-build results now records `build-profile` explicitly + +Validation: + +- `PASS phase20-development-environment-xcpng` +- `PASS phase20-self-hosted-native-build-xcpng` +- `PASS phase20-native-build-store-promotion-xcpng` + +Representative validated metadata included: + +- `build_profile_guest=/run/current-system/build-profile` +- `build_profile=/run/current-system/build-profile` +- `helper_version=5` +- `executor_version=5` + +Report: + +- `docs/reports/postphase20-build-profile-separation-freebsd.md` +- `docs/system-deployment-workflow.md` +- `docs/GUIX_DIFFERENCES.md` + ### 2026-04-06 — Installed systems can now build and reconfigure themselves from local declaration state Fruix-installed systems are now meaningfully closer to real Fruix nodes. diff --git a/docs/reports/postphase20-build-profile-separation-freebsd.md b/docs/reports/postphase20-build-profile-separation-freebsd.md new file mode 100644 index 0000000..5dc88fd --- /dev/null +++ b/docs/reports/postphase20-build-profile-separation-freebsd.md @@ -0,0 +1,211 @@ +# Post-Phase 20: separate development from native base-build environment + +Date: 2026-04-06 + +## Goal + +Tighten the distinction that Phase 20.3 exposed in practice: + +- an interactive development shell is not the same thing as a reliable native base-build environment + +Fruix now models three layers instead of two: + +- runtime profile +- development profile +- build profile + +## Why this change was needed + +The earlier self-hosted native-build prototype proved that simply reusing the exported development environment for `buildworld` / `buildkernel` was too loose. + +Development-oriented exports like: + +- `MAKEFLAGS` +- `CPPFLAGS` +- `CFLAGS` +- `CXXFLAGS` +- `LDFLAGS` + +are useful for interactive compilation work, but they are the wrong contract for a real FreeBSD world/kernel bootstrap. + +Phase 20.3 previously worked around that by manually sanitizing the shell before running the build. + +This change makes that separation explicit in the product model instead of keeping it as an ad hoc helper detail. + +## Implementation + +### New operating-system layer + +`modules/fruix/system/freebsd/model.scm` now supports: + +- `#:build-packages` +- `operating-system-build-packages` + +So a build-capable system can now carry both: + +- `development-packages` +- `build-packages` + +separately. + +### New build profile inside the system closure + +When `build-packages` is non-empty, Fruix now materializes: + +- `/frx/store/...-fruix-system-.../build-profile` + +alongside the existing: + +- `/frx/store/...-fruix-system-.../profile` +- `/frx/store/...-fruix-system-.../development-profile` + +Store-layout metadata now records both development-package stores and build-package stores explicitly. + +### New in-guest build helper + +Build-capable systems now ship: + +- `/usr/local/bin/fruix-build-environment` + +and expose a stable runtime link: + +- `/run/current-build -> /run/current-system/build-profile` + +That helper intentionally emits a stricter environment contract than the interactive development helper. + +It clears development-shell variables such as: + +- `MAKEOBJDIRPREFIX` +- `MAKEFLAGS` +- `CC` +- `CXX` +- `AR` +- `RANLIB` +- `NM` +- `CPPFLAGS` +- `CFLAGS` +- `CXXFLAGS` +- `LDFLAGS` + +and also clears development-helper exports such as: + +- `FRUIX_DEVELOPMENT_PROFILE` +- `FRUIX_CC` +- `FRUIX_CXX` +- `FRUIX_AR` +- `FRUIX_RANLIB` +- `FRUIX_NM` +- `FRUIX_BMAKE` + +Then it exports build-specific values including at least: + +- `FRUIX_BUILD_PROFILE` +- `FRUIX_BUILD_INCLUDE` +- `FRUIX_BUILD_SHARE_MK` +- `FRUIX_BUILD_BIN` +- `FRUIX_BUILD_USR_BIN` +- `FRUIX_BUILD_CC` +- `FRUIX_BUILD_CXX` +- `FRUIX_BUILD_AR` +- `FRUIX_BUILD_RANLIB` +- `FRUIX_BUILD_NM` +- `FRUIX_BMAKE` +- `PATH` + +Intended use: + +```sh +eval "$(/usr/local/bin/fruix-build-environment)" +``` + +### Canonical build compatibility links now come from the build profile + +For native base-build compatibility, build-capable systems now expose: + +- `/usr/include -> /run/current-system/build-profile/usr/include` +- `/usr/share/mk -> /run/current-system/build-profile/usr/share/mk` + +This means the running system can still keep development and build content separate while offering the canonical paths that FreeBSD native build machinery expects. + +### Self-hosted native-build helper now uses the build helper contract + +`/usr/local/bin/fruix-self-hosted-native-build` now: + +- requires `build-profile` +- requires `/usr/local/bin/fruix-build-environment` +- evaluates the build helper contract before `buildworld` +- verifies the canonical compatibility links point at `build-profile` + +This replaces the earlier approach where the helper had to reconstruct sanitization manually around a development-oriented environment. + +### Promotion metadata now records build profile provenance + +Promotion metadata emitted for self-hosted native-build results now records: + +- `build-profile` + +Promoted artifact/result metadata also now preserves that field. + +## Validation + +Validated on the approved real XCP-ng path: + +- VM `90490f2e-e8fc-4b7a-388e-5c26f0157289` +- VDI `0f1f90d3-48ca-4fa2-91d8-fc6339b95743` + +Passing runs: + +- `PASS phase20-development-environment-xcpng` +- `PASS phase20-self-hosted-native-build-xcpng` +- `PASS phase20-native-build-store-promotion-xcpng` + +## Representative validated results + +Development/build environment validation: + +```text +development_profile_guest=/run/current-system/development-profile +build_profile_guest=/run/current-system/build-profile +development_env_script=/frx/store/...-fruix-system-.../usr/local/bin/fruix-development-environment +build_env_script=/frx/store/...-fruix-system-.../usr/local/bin/fruix-build-environment +development_environment=ok +``` + +Self-hosted native-build validation: + +```text +helper_version=5 +executor_kind=self-hosted +executor_name=guest-self-hosted +executor_version=5 +build_profile=/run/current-system/build-profile +source_store=/frx/store/12d7704362e95afc2697db63f168b878e082b372-freebsd-source-default +self_hosted_native_build=ok +``` + +Promotion validation: + +```text +executor_kind=self-hosted +executor_name=guest-self-hosted +executor_version=5 +result_store=/frx/store/3ce6aefd564bc51f2465dcbb5c261355be4c7076-fruix-native-build-result-15.0-STABLE-guest-self-hosted +native_build_store_promotion=ok +``` + +## Result + +Fruix now expresses an important product distinction more honestly: + +- development is for interactive work +- build is for a stricter native base-build contract + +That reduces special-case cleanup inside the self-hosted helper and gives Fruix a clearer path for future operator-facing commands such as: + +- `fruix system build-base` +- `fruix system upgrade` + +because the system model now has an explicit place to say: + +- what is available for interactive development +- what is available for real native base builds \ No newline at end of file diff --git a/docs/system-deployment-workflow.md b/docs/system-deployment-workflow.md index a3257d9..1ded1be 100644 --- a/docs/system-deployment-workflow.md +++ b/docs/system-deployment-workflow.md @@ -250,16 +250,18 @@ fruix system reconfigure /path/to/candidate.scm --system my-operating-system 5. if needed, run `fruix system rollback` 6. reboot back into the recorded prior generation -### In-guest development environment +### In-guest development and build environments -Opt-in systems can also expose a separate development overlay under: +Opt-in systems can now expose two separate overlays above the main runtime profile: -- `/run/current-system/development-profile` -- `/run/current-development` - -Those systems now ship a helper at: - -- `/usr/local/bin/fruix-development-environment` +- development: + - `/run/current-system/development-profile` + - `/run/current-development` + - `/usr/local/bin/fruix-development-environment` +- build: + - `/run/current-system/build-profile` + - `/run/current-build` + - `/usr/local/bin/fruix-build-environment` Intended use: @@ -267,19 +269,53 @@ Intended use: eval "$(/usr/local/bin/fruix-development-environment)" ``` -That helper exports a development-oriented environment while keeping the main runtime profile separate. The validated Phase 20 path currently uses this to expose at least: +for interactive development work, and: + +```sh +eval "$(/usr/local/bin/fruix-build-environment)" +``` + +for a narrower native base-build contract. + +The current split is: + +- runtime profile +- development profile +- build profile + +The development helper remains intentionally interactive and currently exposes at least: - native headers under `usr/include` - FreeBSD `share/mk` files for `bsd.*.mk` - Clang toolchain commands such as `cc`, `c++`, `ar`, `ranlib`, and `nm` - `MAKEFLAGS` pointing at the development profile's `usr/share/mk` -For native base-build compatibility, development-enabled systems also now expose canonical links at: +The build helper is intentionally more sanitized and less interactive. It clears development-shell variables such as: -- `/usr/include -> /run/current-system/development-profile/usr/include` -- `/usr/share/mk -> /run/current-system/development-profile/usr/share/mk` +- `MAKEFLAGS` +- `CPPFLAGS` +- `CFLAGS` +- `CXXFLAGS` +- `LDFLAGS` -This is the current Fruix-native way to make a running system suitable for controlled native base-development work without merging development content back into the main runtime profile. +and then exposes build-oriented paths such as: + +- `FRUIX_BUILD_PROFILE` +- `FRUIX_BUILD_INCLUDE` +- `FRUIX_BUILD_SHARE_MK` +- `FRUIX_BUILD_CC` +- `FRUIX_BUILD_CXX` +- `FRUIX_BUILD_AR` +- `FRUIX_BUILD_RANLIB` +- `FRUIX_BUILD_NM` +- `FRUIX_BMAKE` + +For native base-build compatibility, build-enabled systems now expose canonical links at: + +- `/usr/include -> /run/current-system/build-profile/usr/include` +- `/usr/share/mk -> /run/current-system/build-profile/usr/share/mk` + +So Fruix now separates interactive development support from the stricter environment used for `buildworld` / `buildkernel` style work, instead of treating them as one overlay. ### Host-initiated native base builds inside a Fruix-managed guest @@ -327,7 +363,7 @@ FRUIX_SELF_HOSTED_NATIVE_BUILD_JOBS=8 \ That helper: -1. verifies the development overlay and canonical compatibility links +1. evaluates the build helper and verifies the build overlay plus canonical compatibility links 2. recovers the materialized FreeBSD source store from: - `/run/current-system/metadata/store-layout.scm` 3. runs the native FreeBSD build/install phases inside the guest @@ -344,8 +380,9 @@ That helper: Important current detail: -- the self-hosted helper intentionally **sanitizes** development-shell exports such as `MAKEFLAGS`, `CPPFLAGS`, `CFLAGS`, `CXXFLAGS`, and `LDFLAGS` before `buildworld` -- directly reusing the full development-shell environment polluted FreeBSD's bootstrap path and was not reliable enough for real world/kernel builds +- the self-hosted helper now uses the separate `fruix-build-environment` contract instead of reusing the interactive development helper wholesale +- that build helper intentionally clears development-shell exports such as `MAKEFLAGS`, `CPPFLAGS`, `CFLAGS`, `CXXFLAGS`, and `LDFLAGS` before `buildworld` +- this keeps the base-build path closer to the exact contract needed for real world/kernel bootstrap work So the validated Phase 20.3 answer is: diff --git a/modules/fruix/system/freebsd.scm b/modules/fruix/system/freebsd.scm index 18ce904..4551483 100644 --- a/modules/fruix/system/freebsd.scm +++ b/modules/fruix/system/freebsd.scm @@ -40,6 +40,7 @@ operating-system-bootloader operating-system-base-packages operating-system-development-packages + operating-system-build-packages operating-system-users operating-system-groups operating-system-file-systems diff --git a/modules/fruix/system/freebsd/build.scm b/modules/fruix/system/freebsd/build.scm index 6d87162..978d1ee 100644 --- a/modules/fruix/system/freebsd/build.scm +++ b/modules/fruix/system/freebsd/build.scm @@ -720,7 +720,9 @@ (define (native-build-promoted-artifact-metadata result artifact-kind content-signature) (let* ((entry (native-build-artifact-entry result artifact-kind)) - (executor (native-build-result-executor result))) + (executor (native-build-result-executor result)) + (build-profile (native-build-result-ref result 'build-profile + (native-build-result-ref result 'development-profile "")))) `((native-build-object-version . ,native-build-result-promotion-version) (object-kind . artifact) (artifact-kind . ,artifact-kind) @@ -732,6 +734,7 @@ (guest-host-name . ,(native-build-result-ref result 'guest-host-name "unknown")) (closure-path . ,(native-build-result-ref result 'closure-path "")) (development-profile . ,(native-build-result-ref result 'development-profile "")) + (build-profile . ,build-profile) (freebsd-base . ,(native-build-result-ref result 'freebsd-base '())) (source . ,(native-build-result-ref result 'source '())) (build-policy . ,(native-build-result-ref result 'build-policy '())) @@ -769,7 +772,9 @@ (string-append "fruix-native-build-result-" version-label "-" executor-name))) (define (native-build-promoted-result-object result promoted-artifacts) - (let ((executor (native-build-result-executor result))) + (let ((executor (native-build-result-executor result)) + (build-profile (native-build-result-ref result 'build-profile + (native-build-result-ref result 'development-profile "")))) `((native-build-result-version . ,native-build-result-promotion-version) (object-kind . result-bundle) (executor . ,executor) @@ -780,6 +785,7 @@ (guest-host-name . ,(native-build-result-ref result 'guest-host-name "unknown")) (closure-path . ,(native-build-result-ref result 'closure-path "")) (development-profile . ,(native-build-result-ref result 'development-profile "")) + (build-profile . ,build-profile) (freebsd-base . ,(native-build-result-ref result 'freebsd-base '())) (source . ,(native-build-result-ref result 'source '())) (build-policy . ,(native-build-result-ref result 'build-policy '())) diff --git a/modules/fruix/system/freebsd/media.scm b/modules/fruix/system/freebsd/media.scm index f121cb5..29949a4 100644 --- a/modules/fruix/system/freebsd/media.scm +++ b/modules/fruix/system/freebsd/media.scm @@ -87,6 +87,7 @@ (bootloader-package (operating-system-bootloader os)) (base-packages (operating-system-base-packages os)) (development-packages (operating-system-development-packages os)) + (build-packages (operating-system-build-packages os)) (kernel-store (materialize-freebsd-package kernel-package store-dir cache source-cache)) (bootloader-store (materialize-freebsd-package bootloader-package store-dir cache source-cache)) (base-package-stores (map (lambda (package) @@ -96,6 +97,10 @@ (map (lambda (package) (materialize-freebsd-package package store-dir cache source-cache)) development-packages)) + (build-package-stores + (map (lambda (package) + (materialize-freebsd-package package store-dir cache source-cache)) + build-packages)) (base-package-pairs (map cons base-packages base-package-stores)) (store-classification (append (list (cons kernel-package kernel-store) @@ -204,6 +209,8 @@ (native-base-stores . ,native-base-stores) (development-package-store-count . ,(length development-package-stores)) (development-package-stores . ,development-package-stores) + (build-package-store-count . ,(length build-package-stores)) + (build-package-stores . ,build-package-stores) (fruix-runtime-store-count . ,(length fruix-runtime-stores)) (fruix-runtime-stores . ,fruix-runtime-stores) (host-base-replacement-order . ,%freebsd-host-staged-replacement-order) @@ -229,6 +236,7 @@ host-base-stores native-base-stores development-package-stores + build-package-stores fruix-runtime-stores))) (manifest (string-append "closure-spec=\n" @@ -245,7 +253,9 @@ (closure-path (make-store-path store-dir display-name manifest #:kind 'operating-system)) (development-profile-path (and (not (null? development-package-stores)) - (string-append closure-path "/development-profile")))) + (string-append closure-path "/development-profile"))) + (build-profile-path (and (not (null? build-package-stores)) + (string-append closure-path "/build-profile")))) (unless (file-exists? closure-path) (mkdir-p closure-path) (mkdir-p (string-append closure-path "/boot/kernel")) @@ -268,6 +278,11 @@ (for-each (lambda (output) (merge-output-into-tree output development-profile-path)) development-package-stores)) + (when build-profile-path + (mkdir-p build-profile-path) + (for-each (lambda (output) + (merge-output-into-tree output build-profile-path)) + build-package-stores)) (for-each (lambda (entry) (write-file (string-append closure-path "/" (car entry)) (cdr entry))) @@ -281,6 +296,8 @@ (chmod (string-append closure-path "/usr/local/bin/fruix") #o555)) (when (file-exists? (string-append closure-path "/usr/local/bin/fruix-development-environment")) (chmod (string-append closure-path "/usr/local/bin/fruix-development-environment") #o555)) + (when (file-exists? (string-append closure-path "/usr/local/bin/fruix-build-environment")) + (chmod (string-append closure-path "/usr/local/bin/fruix-build-environment") #o555)) (when (file-exists? (string-append closure-path "/usr/local/bin/fruix-self-hosted-native-build")) (chmod (string-append closure-path "/usr/local/bin/fruix-self-hosted-native-build") #o555)) (when (file-exists? (string-append closure-path "/boot/fruix-pid1")) @@ -298,7 +315,9 @@ (shepherd-store . ,shepherd-store) (base-package-stores . ,base-package-stores) (development-package-stores . ,development-package-stores) + (build-package-stores . ,build-package-stores) (development-profile-path . ,development-profile-path) + (build-profile-path . ,build-profile-path) (host-base-stores . ,host-base-stores) (native-base-stores . ,native-base-stores) (fruix-runtime-stores . ,fruix-runtime-stores) @@ -446,7 +465,18 @@ (string-append rootfs "/usr/local/bin/fruix")) (when (file-exists? (string-append closure-path "/development-profile")) (symlink-force "/run/current-system/development-profile" - (string-append rootfs "/run/current-development")) + (string-append rootfs "/run/current-development"))) + (when (file-exists? (string-append closure-path "/build-profile")) + (symlink-force "/run/current-system/build-profile" + (string-append rootfs "/run/current-build")) + (when (file-exists? (string-append closure-path "/build-profile/usr/include")) + (symlink-force "/run/current-system/build-profile/usr/include" + (string-append rootfs "/usr/include"))) + (when (file-exists? (string-append closure-path "/build-profile/usr/share/mk")) + (symlink-force "/run/current-system/build-profile/usr/share/mk" + (string-append rootfs "/usr/share/mk")))) + (when (and (not (file-exists? (string-append closure-path "/build-profile"))) + (file-exists? (string-append closure-path "/development-profile"))) (when (file-exists? (string-append closure-path "/development-profile/usr/include")) (symlink-force "/run/current-system/development-profile/usr/include" (string-append rootfs "/usr/include"))) @@ -456,6 +486,9 @@ (when (file-exists? (string-append closure-path "/usr/local/bin/fruix-development-environment")) (symlink-force "/run/current-system/usr/local/bin/fruix-development-environment" (string-append rootfs "/usr/local/bin/fruix-development-environment"))) + (when (file-exists? (string-append closure-path "/usr/local/bin/fruix-build-environment")) + (symlink-force "/run/current-system/usr/local/bin/fruix-build-environment" + (string-append rootfs "/usr/local/bin/fruix-build-environment"))) (when (file-exists? (string-append closure-path "/usr/local/bin/fruix-self-hosted-native-build")) (symlink-force "/run/current-system/usr/local/bin/fruix-self-hosted-native-build" (string-append rootfs "/usr/local/bin/fruix-self-hosted-native-build"))) diff --git a/modules/fruix/system/freebsd/model.scm b/modules/fruix/system/freebsd/model.scm index 59375d6..416f97c 100644 --- a/modules/fruix/system/freebsd/model.scm +++ b/modules/fruix/system/freebsd/model.scm @@ -41,6 +41,7 @@ operating-system-bootloader operating-system-base-packages operating-system-development-packages + operating-system-build-packages operating-system-users operating-system-groups operating-system-file-systems @@ -155,7 +156,7 @@ (define-record-type (make-operating-system host-name freebsd-base native-build-result kernel bootloader - base-packages development-packages users groups file-systems + base-packages development-packages build-packages users groups file-systems services loader-entries rc-conf-entries init-mode ready-marker root-authorized-keys) operating-system? @@ -166,6 +167,7 @@ (bootloader operating-system-bootloader) (base-packages operating-system-base-packages) (development-packages operating-system-development-packages) + (build-packages operating-system-build-packages) (users operating-system-users) (groups operating-system-groups) (file-systems operating-system-file-systems) @@ -184,6 +186,7 @@ (bootloader freebsd-bootloader) (base-packages %freebsd-system-packages) (development-packages '()) + (build-packages '()) (users (list (user-account #:name "root" #:uid 0 #:group "wheel" @@ -226,7 +229,7 @@ (ready-marker "/var/lib/fruix/ready") (root-authorized-keys '())) (make-operating-system host-name freebsd-base native-build-result kernel bootloader - base-packages development-packages users groups file-systems + base-packages development-packages build-packages users groups file-systems services loader-entries rc-conf-entries init-mode ready-marker root-authorized-keys)) @@ -299,6 +302,7 @@ (native-build-result (operating-system-native-build-result os)) (base-packages (operating-system-base-packages os)) (development-packages (operating-system-development-packages os)) + (build-packages (operating-system-build-packages os)) (users (operating-system-users os)) (groups (operating-system-groups os)) (file-systems (operating-system-file-systems os)) @@ -317,6 +321,8 @@ (error "operating-system base-packages must be a list of records")) (unless (every freebsd-package? development-packages) (error "operating-system development-packages must be a list of records")) + (unless (every freebsd-package? build-packages) + (error "operating-system build-packages must be a list of records")) (validate-freebsd-source (freebsd-base-source base)) (let ((dups (duplicate-elements user-names))) (unless (null? dups) @@ -381,7 +387,10 @@ '()) (if (null? (operating-system-development-packages os)) '() - '("usr/local/bin/fruix-development-environment" + '("usr/local/bin/fruix-development-environment")) + (if (null? (operating-system-build-packages os)) + '() + '("usr/local/bin/fruix-build-environment" "usr/local/bin/fruix-self-hosted-native-build")) (if (pid1-init-mode? os) '("boot/fruix-pid1") @@ -408,12 +417,16 @@ (base-packages . ,(package-names (operating-system-base-packages os))) (development-package-count . ,(length (operating-system-development-packages os))) (development-packages . ,(package-names (operating-system-development-packages os))) + (build-package-count . ,(length (operating-system-build-packages os))) + (build-packages . ,(package-names (operating-system-build-packages os))) (installed-system-command-surface-version . "2") (bundled-fruix-node-cli-version . "1") (development-environment-helper-version . ,(if (null? (operating-system-development-packages os)) #f "1")) + (build-environment-helper-version + . ,(if (null? (operating-system-build-packages os)) #f "1")) (self-hosted-native-build-helper-version - . ,(if (null? (operating-system-development-packages os)) #f "4")) + . ,(if (null? (operating-system-build-packages os)) #f "5")) (user-count . ,(length (operating-system-users os))) (users . ,(map user-account-name (operating-system-users os))) (group-count . ,(length (operating-system-groups os))) diff --git a/modules/fruix/system/freebsd/render.scm b/modules/fruix/system/freebsd/render.scm index 38aa05f..7e32099 100644 --- a/modules/fruix/system/freebsd/render.scm +++ b/modules/fruix/system/freebsd/render.scm @@ -934,6 +934,33 @@ "export MAKEFLAGS=\"-m $profile/usr/share/mk\"\n" "export PATH=\"/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin:$profile/bin:$profile/sbin:$profile/usr/bin:$profile/usr/sbin\"\n" "EOF\n")) +(define (render-build-environment-script os) + (string-append + "#!/bin/sh\n" + "set -eu\n" + "profile=/run/current-system/build-profile\n" + "[ -d \"$profile\" ] || {\n" + " echo \"fruix-build-environment: build profile is not available\" >&2\n" + " exit 1\n" + "}\n" + "cat <&2\n" + " echo \"fruix-self-hosted-native-build: build profile is not available\" >&2\n" " exit 1\n" "}\n" - "[ -x /usr/local/bin/fruix-development-environment ] || {\n" - " echo \"fruix-self-hosted-native-build: development environment helper is missing\" >&2\n" + "[ -x /usr/local/bin/fruix-build-environment ] || {\n" + " echo \"fruix-self-hosted-native-build: build environment helper is missing\" >&2\n" + " exit 1\n" + "}\n" + "eval \"$(/usr/local/bin/fruix-build-environment)\"\n" + "[ \"${FRUIX_BUILD_PROFILE:-}\" = \"$profile\" ] || {\n" + " echo \"fruix-self-hosted-native-build: build environment helper exported an unexpected profile\" >&2\n" " exit 1\n" "}\n" - "PATH=/bin:/sbin:/usr/bin:/usr/sbin:/usr/local/bin:/usr/local/sbin\n" - "unset MAKEOBJDIRPREFIX MAKEFLAGS CC CXX AR RANLIB NM CPPFLAGS CFLAGS CXXFLAGS LDFLAGS\n" - "unset FRUIX_DEVELOPMENT_PROFILE FRUIX_DEVELOPMENT_INCLUDE FRUIX_DEVELOPMENT_LIB FRUIX_DEVELOPMENT_SHARE_MK\n" - "unset FRUIX_DEVELOPMENT_BIN FRUIX_DEVELOPMENT_USR_BIN FRUIX_CC FRUIX_CXX FRUIX_AR FRUIX_RANLIB FRUIX_NM FRUIX_BMAKE\n" "[ -L /usr/include ] || {\n" " echo \"fruix-self-hosted-native-build: /usr/include compatibility link is missing\" >&2\n" " exit 1\n" "}\n" - "[ \"$(readlink /usr/include)\" = \"/run/current-system/development-profile/usr/include\" ] || {\n" + "[ \"$(readlink /usr/include)\" = \"/run/current-system/build-profile/usr/include\" ] || {\n" " echo \"fruix-self-hosted-native-build: /usr/include points at the wrong target\" >&2\n" " exit 1\n" "}\n" @@ -982,10 +1010,11 @@ " echo \"fruix-self-hosted-native-build: /usr/share/mk compatibility link is missing\" >&2\n" " exit 1\n" "}\n" - "[ \"$(readlink /usr/share/mk)\" = \"/run/current-system/development-profile/usr/share/mk\" ] || {\n" + "[ \"$(readlink /usr/share/mk)\" = \"/run/current-system/build-profile/usr/share/mk\" ] || {\n" " echo \"fruix-self-hosted-native-build: /usr/share/mk points at the wrong target\" >&2\n" " exit 1\n" "}\n" + "make_cmd=${FRUIX_BMAKE:-make}\n" "jobs=${FRUIX_SELF_HOSTED_NATIVE_BUILD_JOBS:-$(sysctl -n hw.ncpu)}\n" "case \"$jobs\" in\n" " ''|*[!0-9]*)\n" @@ -1036,11 +1065,11 @@ "}\n" "mkdir -p \"$world_artifact\" \"$headers_artifact/usr\" \"$kernel_artifact/boot\" \"$bootloader_artifact/boot\"\n" "export MAKEOBJDIRPREFIX=\"$build_root/obj\"\n" - "make -j\"$jobs\" -C \"$source_root\" " build-common " buildworld > \"$logdir/buildworld.log\" 2>&1\n" - "make -j\"$jobs\" -C \"$source_root\" " build-common " buildkernel > \"$logdir/buildkernel.log\" 2>&1\n" - "make -C \"$source_root\" " install-common " DESTDIR=\"$world_stage\" installworld > \"$logdir/installworld.log\" 2>&1\n" - "make -C \"$source_root\" " install-common " DESTDIR=\"$world_stage\" distribution > \"$logdir/distribution.log\" 2>&1\n" - "make -C \"$source_root\" " install-common " DESTDIR=\"$kernel_stage\" installkernel > \"$logdir/installkernel.log\" 2>&1\n" + "\"$make_cmd\" -j\"$jobs\" -C \"$source_root\" " build-common " buildworld > \"$logdir/buildworld.log\" 2>&1\n" + "\"$make_cmd\" -j\"$jobs\" -C \"$source_root\" " build-common " buildkernel > \"$logdir/buildkernel.log\" 2>&1\n" + "\"$make_cmd\" -C \"$source_root\" " install-common " DESTDIR=\"$world_stage\" installworld > \"$logdir/installworld.log\" 2>&1\n" + "\"$make_cmd\" -C \"$source_root\" " install-common " DESTDIR=\"$world_stage\" distribution > \"$logdir/distribution.log\" 2>&1\n" + "\"$make_cmd\" -C \"$source_root\" " install-common " DESTDIR=\"$kernel_stage\" installkernel > \"$logdir/installkernel.log\" 2>&1\n" "cp -a \"$world_stage/.\" \"$world_artifact/\"\n" "cp -a \"$kernel_stage/boot/kernel\" \"$kernel_artifact/boot/kernel\"\n" "cp -a \"$world_stage/usr/include\" \"$headers_artifact/usr/include\"\n" @@ -1079,7 +1108,7 @@ "((native-build-result-version . \"1\")\n" " (executor . ((kind . self-hosted)\n" " (name . \"guest-self-hosted\")\n" - " (version . \"4\")\n" + " (version . \"5\")\n" " (properties . ((helper-path . \"/usr/local/bin/fruix-self-hosted-native-build\")\n" " (guest-host-name . \"$guest_host_name\")\n" " (build-root-base . \"$build_root_base\")\n" @@ -1087,7 +1116,7 @@ " (run-id . \"$run_id\")\n" " (guest-host-name . \"$guest_host_name\")\n" " (closure-path . \"$closure\")\n" - " (development-profile . \"$profile\")\n" + " (build-profile . \"$profile\")\n" " (freebsd-base . ((name . \"" base-name "\")\n" " (version-label . \"" version-label "\")\n" " (release . \"" release "\")\n" @@ -1115,13 +1144,13 @@ "EOF\n" "cat >\"$metadata_file\" <&2; exit 1; } [ "$sshd_status" = running ] || { echo "sshd is not running" >&2; exit 1; } @@ -68,7 +70,11 @@ for path in \ "$development_profile_path/bin/ar" \ "$development_profile_path/usr/include/sys/param.h" \ "$development_profile_path/usr/share/mk/bsd.prog.mk" \ - "$development_env_script" + "$build_profile_path/bin/cc" \ + "$build_profile_path/usr/include/sys/param.h" \ + "$build_profile_path/usr/share/mk/bsd.prog.mk" \ + "$development_env_script" \ + "$build_env_script" do [ -e "$path" ] || { echo "required development environment path missing: $path" >&2 @@ -96,8 +102,11 @@ guest_dev_metadata=$(ssh -i "$root_ssh_private_key_file" \ root@"$guest_ip" 'sh -s' <<'EOF' set -eu [ -x /usr/local/bin/fruix-development-environment ] +[ -x /usr/local/bin/fruix-build-environment ] [ -L /run/current-development ] [ "$(readlink /run/current-development)" = "/run/current-system/development-profile" ] +[ -L /run/current-build ] +[ "$(readlink /run/current-build)" = "/run/current-system/build-profile" ] exports=$(/usr/local/bin/fruix-development-environment) printf '%s\n' "$exports" | grep '^export FRUIX_DEVELOPMENT_PROFILE="/run/current-system/development-profile"$' >/dev/null printf '%s\n' "$exports" | grep '^export MAKEFLAGS="-m /run/current-system/development-profile/usr/share/mk"$' >/dev/null @@ -109,6 +118,20 @@ eval "$exports" [ -f "$FRUIX_DEVELOPMENT_INCLUDE/sys/param.h" ] [ -f "$FRUIX_DEVELOPMENT_SHARE_MK/bsd.prog.mk" ] cc_version=$($FRUIX_CC --version | awk 'NR==1 { print; exit }') +build_exports=$(/usr/local/bin/fruix-build-environment) +printf '%s\n' "$build_exports" | grep '^unset MAKEOBJDIRPREFIX MAKEFLAGS CC CXX AR RANLIB NM CPPFLAGS CFLAGS CXXFLAGS LDFLAGS$' >/dev/null +printf '%s\n' "$build_exports" | grep '^export FRUIX_BUILD_PROFILE="/run/current-system/build-profile"$' >/dev/null +eval "$build_exports" +[ -d "$FRUIX_BUILD_PROFILE" ] +[ -f "$FRUIX_BUILD_INCLUDE/sys/param.h" ] +[ -f "$FRUIX_BUILD_SHARE_MK/bsd.prog.mk" ] +[ "${MAKEFLAGS-unset}" = unset ] +[ "${CPPFLAGS-unset}" = unset ] +[ "${CFLAGS-unset}" = unset ] +[ "${LDFLAGS-unset}" = unset ] +[ "$FRUIX_BMAKE" = make ] +build_profile_value=$FRUIX_BUILD_PROFILE +eval "$exports" tmp=/tmp/fruix-phase20-development-env rm -rf "$tmp" mkdir -p "$tmp/direct" "$tmp/mk" @@ -142,25 +165,34 @@ hello_make=$(./hello) make_log_tail=$(tail -n 20 "$tmp/mk/make.log" | tr '\n' ' ') exports_flat=$(printf '%s' "$exports" | tr '\n' ' ') printf 'development_profile=%s\n' "$FRUIX_DEVELOPMENT_PROFILE" +printf 'build_profile=%s\n' "$build_profile_value" printf 'cc_version=%s\n' "$cc_version" printf 'hello_direct=%s\n' "$hello_direct" printf 'hello_make=%s\n' "$hello_make" +build_exports_flat=$(printf '%s' "$build_exports" | tr '\n' ' ') printf 'exports=%s\n' "$exports_flat" +printf 'build_exports=%s\n' "$build_exports_flat" printf 'make_log_tail=%s\n' "$make_log_tail" EOF ) development_profile=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^development_profile=//p') +build_profile=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^build_profile=//p') cc_version=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^cc_version=//p') hello_direct=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^hello_direct=//p') hello_make=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^hello_make=//p') development_exports=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^exports=//p') +build_exports=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^build_exports=//p') make_log_tail=$(printf '%s\n' "$guest_dev_metadata" | sed -n 's/^make_log_tail=//p') [ "$development_profile" = "/run/current-system/development-profile" ] || { echo "unexpected guest development profile path: $development_profile" >&2 exit 1 } +[ "$build_profile" = "/run/current-system/build-profile" ] || { + echo "unexpected guest build profile path: $build_profile" >&2 + exit 1 +} case "$cc_version" in *"FreeBSD clang version"*) : ;; *) echo "unexpected cc version output: $cc_version" >&2; exit 1 ;; @@ -177,6 +209,10 @@ case "$development_exports" in *'export FRUIX_CC="/run/current-system/development-profile/bin/cc"'*) : ;; *) echo "development environment exports do not include FRUIX_CC" >&2; exit 1 ;; esac +case "$build_exports" in + *'export FRUIX_BUILD_PROFILE="/run/current-system/build-profile"'*) : ;; + *) echo "build environment exports do not include FRUIX_BUILD_PROFILE" >&2; exit 1 ;; +esac cat >"$metadata_file" <"$promotion_file" <&2; exit 1; } [ "$executor_name" = guest-self-hosted ] || { echo "unexpected executor name: $executor_name" >&2; exit 1; } -[ "$executor_version" = 4 ] || { echo "unexpected executor version: $executor_version" >&2; exit 1; } +[ "$executor_version" = 5 ] || { echo "unexpected executor version: $executor_version" >&2; exit 1; } [ "$artifact_store_count" = 4 ] || { echo "unexpected artifact store count: $artifact_store_count" >&2; exit 1; } case "$result_store" in /frx/store/*-fruix-native-build-result-*-guest-self-hosted) : ;; @@ -178,6 +178,10 @@ grep -F "$source_store" "$result_metadata_file" >/dev/null || { echo "result metadata file is missing source store provenance" >&2 exit 1 } +grep -F '(build-profile . "/run/current-system/build-profile")' "$result_metadata_file" >/dev/null || { + echo "result metadata file is missing build-profile provenance" >&2 + exit 1 +} grep -F '(artifact-kind . kernel)' "$kernel_store/.fruix-native-build-object.scm" >/dev/null || { echo "kernel store metadata is missing artifact kind" >&2 exit 1 diff --git a/tests/system/run-phase20-self-hosted-native-build-xcpng.sh b/tests/system/run-phase20-self-hosted-native-build-xcpng.sh index 4acc787..29ec33a 100755 --- a/tests/system/run-phase20-self-hosted-native-build-xcpng.sh +++ b/tests/system/run-phase20-self-hosted-native-build-xcpng.sh @@ -70,7 +70,9 @@ case "$guest_build_jobs" in ;; esac +ssh_guest '[ -x /usr/local/bin/fruix-build-environment ]' ssh_guest '[ -x /usr/local/bin/fruix-self-hosted-native-build ]' +ssh_guest '[ -L /run/current-build ]' ssh_guest '[ -L /usr/include ]' ssh_guest '[ -L /usr/share/mk ]' @@ -81,6 +83,7 @@ helper_version=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^helper_versi executor_kind=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^executor_kind=//p') executor_name=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^executor_name=//p') executor_version=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^executor_version=//p') +build_profile=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^build_profile=//p') source_store=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^source_store=//p') source_root=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^source_root=//p') build_jobs=$(printf '%s\n' "$self_hosted_metadata" | sed -n 's/^build_jobs=//p') @@ -121,10 +124,11 @@ latest_target=$(ssh_guest "readlink '$latest_link'") ssh_guest "[ -f '$promotion_file' ]" ssh_guest "[ -f '$world_artifact/bin/sh' ]" -[ "$helper_version" = 4 ] || { echo "unexpected helper version: $helper_version" >&2; exit 1; } +[ "$helper_version" = 5 ] || { echo "unexpected helper version: $helper_version" >&2; exit 1; } [ "$executor_kind" = self-hosted ] || { echo "unexpected executor kind: $executor_kind" >&2; exit 1; } [ "$executor_name" = guest-self-hosted ] || { echo "unexpected executor name: $executor_name" >&2; exit 1; } -[ "$executor_version" = 4 ] || { echo "unexpected executor version: $executor_version" >&2; exit 1; } +[ "$executor_version" = 5 ] || { echo "unexpected executor version: $executor_version" >&2; exit 1; } +[ "$build_profile" = /run/current-system/build-profile ] || { echo "unexpected build profile: $build_profile" >&2; exit 1; } [ "$build_jobs" = "$guest_build_jobs" ] || { echo "unexpected build job count: $build_jobs" >&2; exit 1; } [ "$status_value" = ok ] || { echo "self-hosted build status is not ok: $status_value" >&2; exit 1; } [ "$latest_target" = "$result_root" ] || { echo "latest link target mismatch: $latest_target" >&2; exit 1; } @@ -207,6 +211,7 @@ executor_kind=$executor_kind executor_name=$executor_name executor_version=$executor_version build_jobs=$build_jobs +build_profile=$build_profile source_store=$source_store source_root=$source_root build_common=$build_common