From 006ffee615e177a40fb1ac6d82535acf48f7fe7e Mon Sep 17 00:00:00 2001 From: Steffen Beyer Date: Mon, 6 Apr 2026 01:16:44 +0200 Subject: [PATCH] native-build: promote results into store objects --- docs/GUIX_DIFFERENCES.md | 20 ++ docs/PROGRESS.md | 60 +++-- ...20-native-build-store-promotion-freebsd.md | 190 ++++++++++++++ docs/system-deployment-workflow.md | 63 ++++- modules/fruix/system/freebsd.scm | 2 + modules/fruix/system/freebsd/build.scm | 177 ++++++++++++++ modules/fruix/system/freebsd/model.scm | 2 +- modules/fruix/system/freebsd/render.scm | 52 +++- modules/fruix/system/freebsd/utils.scm | 25 ++ scripts/fruix.scm | 57 +++++ ...se20-native-build-store-promotion-xcpng.sh | 231 ++++++++++++++++++ ...-phase20-self-hosted-native-build-xcpng.sh | 10 +- 12 files changed, 863 insertions(+), 26 deletions(-) create mode 100644 docs/reports/phase20-native-build-store-promotion-freebsd.md create mode 100755 tests/system/run-phase20-native-build-store-promotion-xcpng.sh diff --git a/docs/GUIX_DIFFERENCES.md b/docs/GUIX_DIFFERENCES.md index 22b0ea8..322f40b 100644 --- a/docs/GUIX_DIFFERENCES.md +++ b/docs/GUIX_DIFFERENCES.md @@ -274,6 +274,26 @@ The practical Fruix takeaway is: - the development overlay makes native base work possible inside the system - but real guest self-hosted base builds still need their own stricter build contract +Fruix now also makes an explicit distinction between: + +- mutable native-build staging/results under: + - `/var/lib/fruix/native-builds` +- immutable promoted native-build identities under: + - `/frx/store` + +So a guest self-hosted run can stage a result tree locally, while the host can later promote that result into first-class Fruix store objects such as: + +- `/frx/store/...-fruix-native-world-...` +- `/frx/store/...-fruix-native-kernel-...` +- `/frx/store/...-fruix-native-headers-...` +- `/frx/store/...-fruix-native-bootloader-...` +- `/frx/store/...-fruix-native-build-result-...` + +Compared with Guix, this is a more explicit split between: + +- a mutable result/staging area for native build execution +- and the immutable store identities that Fruix treats as the real promoted result + ## Where Fruix is intentionally trying to improve on Guix's representation Fruix is not trying to improve on Guix's core semantics. Guix already got those right. diff --git a/docs/PROGRESS.md b/docs/PROGRESS.md index 4d6170a..b69376a 100644 --- a/docs/PROGRESS.md +++ b/docs/PROGRESS.md @@ -46,6 +46,13 @@ Fruix currently has: - `/usr/local/bin/fruix-self-hosted-native-build` - result roots under `/var/lib/fruix/native-builds` - in-guest source recovery from current-system metadata +- a validated promotion path that turns native-build result roots into first-class Fruix store objects via: + - `fruix native-build promote` + - `/frx/store/...-fruix-native-world-...` + - `/frx/store/...-fruix-native-kernel-...` + - `/frx/store/...-fruix-native-headers-...` + - `/frx/store/...-fruix-native-bootloader-...` + - `/frx/store/...-fruix-native-build-result-...` Validated boot modes still are: @@ -58,32 +65,39 @@ The validated Phase 18 installation work currently uses: ## Latest completed achievement -### 2026-04-05 — Phase 20.3 completed +### 2026-04-05 — Native base builds promoted into first-class Fruix store objects -Fruix now has a validated first controlled guest self-hosted native base-build prototype, on top of the already validated host-initiated in-guest build path. +Fruix now has a validated end-to-end path that treats guest native base-build results as **staged mutable results first**, and then promotes them into **immutable Fruix store identities**. Highlights: -- development-enabled systems now ship an in-guest helper at: - - `/usr/local/bin/fruix-self-hosted-native-build` -- the helper recovers the declared materialized FreeBSD source store from: - - `/run/current-system/metadata/store-layout.scm` -- the helper records self-hosted build results under: +- guest self-hosted runs still record staged results under: - `/var/lib/fruix/native-builds/` - `/var/lib/fruix/native-builds/latest` -- heavy object/stage work remains under: - - `/var/tmp/fruix-self-hosted-native-builds/` -- the prototype exposed an important contract detail: - - a naive reuse of the development-shell exports polluted `buildworld` - - the validated helper therefore sanitizes development-oriented variables such as `MAKEFLAGS`, `CPPFLAGS`, `CFLAGS`, `CXXFLAGS`, and `LDFLAGS` before world/kernel bootstrap -- the validated result includes guest-recorded native artifact outputs for: - - kernel - - bootloader slice - - headers / `usr/share/mk` +- those result roots now carry promotion metadata describing: + - executor / executor-version + - closure path + - source store provenance + - build policy + - artifact entries for: + - `world` + - `kernel` + - `headers` + - `bootloader` +- the host can now run: + - `fruix native-build promote RESULT_ROOT` +- promotion creates immutable `/frx/store` objects for: + - `world` + - `kernel` + - `headers` + - `bootloader` +- promotion also creates a result-bundle store object that references those artifact stores +- the validated promotion metadata now makes Fruix-native native-build identity explicit instead of leaving results only as ad hoc files under `/var/lib/fruix/native-builds/...` Validation: - `PASS phase20-self-hosted-native-build-xcpng` +- `PASS phase20-native-build-store-promotion-xcpng` Reports: @@ -92,6 +106,7 @@ Reports: - `docs/reports/phase20-development-environment-freebsd.md` - `docs/reports/phase20-host-initiated-native-builds-freebsd.md` - `docs/reports/phase20-self-hosted-native-builds-freebsd.md` +- `docs/reports/phase20-native-build-store-promotion-freebsd.md` ## Recent major milestones @@ -117,9 +132,14 @@ Reports: `docs/PLAN_4.md` currently ends at Phase 20.3, and that milestone sequence is now complete. -The next practical follow-up is therefore no longer another planned `PLAN_4` phase, but a product decision: +The next practical follow-up is now clearer: -- whether to keep the new guest self-hosted helper as a narrow prototype while Phase 20.2 remains the default operator path -- or whether to invest in a broader guest-driven Fruix-native native-build workflow +- unify host-initiated and self-hosted native-build execution behind a shared Fruix executor/result model +- make the same first-class promotion story available regardless of whether the outer loop is host-driven or guest-driven +- decide how much of result import/promotion should remain host-side versus become a more integrated Fruix deployment action -Phase 20.3 is now complete: Fruix validates a first controlled guest self-hosted native FreeBSD base-build prototype on the approved real XCP-ng path. +The immediate architectural direction is no longer just “can guest self-hosting work?” + +It is now: + +- how should Fruix represent native base builds as real first-class objects and workflows across different executors? diff --git a/docs/reports/phase20-native-build-store-promotion-freebsd.md b/docs/reports/phase20-native-build-store-promotion-freebsd.md new file mode 100644 index 0000000..40d0036 --- /dev/null +++ b/docs/reports/phase20-native-build-store-promotion-freebsd.md @@ -0,0 +1,190 @@ +# Post-Phase 20: native build result promotion into first-class Fruix store objects + +Date: 2026-04-05 + +## Goal + +Make native FreeBSD base-build results feel like real Fruix objects instead of stopping at mutable staged files under: + +- `/var/lib/fruix/native-builds/...` + +The desired model is: + +- `/var/lib/fruix/native-builds/...` remains a staging/result area +- `/frx/store/...` remains the real immutable identity + +Validated artifact identities: + +- `world` +- `kernel` +- `headers` +- `bootloader` + +## What changed + +### Promotion metadata in guest result roots + +The guest self-hosted helper now emits a promotion description file at: + +- `/var/lib/fruix/native-builds//promotion.scm` + +That metadata records at least: + +- executor / executor-version +- run-id / guest-host-name +- closure path +- development profile path +- declared FreeBSD base metadata +- source store provenance +- build policy +- artifact entries for: + - `world` + - `kernel` + - `headers` + - `bootloader` + +The helper also stages a promotable `world` artifact tree in addition to the already validated narrower artifacts. + +### Host-side promotion API + +Fruix now exports: + +- `promote-native-build-result` + +and the CLI now exposes: + +- `fruix native-build promote RESULT_ROOT [--store DIR]` + +### Promoted store object layout + +Promotion now creates immutable store objects for: + +- `/frx/store/...-fruix-native-world-...` +- `/frx/store/...-fruix-native-kernel-...` +- `/frx/store/...-fruix-native-headers-...` +- `/frx/store/...-fruix-native-bootloader-...` + +Each promoted artifact store records: + +- `.fruix-native-build-object.scm` +- `.references` + +Promotion also creates a result-bundle store object: + +- `/frx/store/...-fruix-native-build-result-...` + +That bundle records: + +- `.fruix-native-build-result.scm` +- `artifacts/world` +- `artifacts/kernel` +- `artifacts/headers` +- `artifacts/bootloader` + +where the `artifacts/*` entries are symlinks to the promoted artifact stores. + +### Identity policy + +Artifact identity is now based on Fruix metadata plus a tree-content signature of the staged artifact tree. + +That means promotion identity depends on both: + +- the explicit Fruix-native build metadata +- the actual content of the promoted artifact tree + +## Validation harness + +Added: + +- `tests/system/run-phase20-native-build-store-promotion-xcpng.sh` + +This harness: + +1. boots the approved real XCP-ng guest path +2. runs the validated in-guest self-hosted native build helper +3. imports the guest result root back to the host +4. runs `fruix native-build promote` +5. verifies promoted store paths, metadata, symlink structure, and representative hashes + +## Validation + +Passing run: + +- `PASS phase20-native-build-store-promotion-xcpng` +- workdir: `/tmp/current-phase20-native-build-store-promotion-xcpng` + +Approved real XCP-ng path: + +- VM `90490f2e-e8fc-4b7a-388e-5c26f0157289` +- VDI `0f1f90d3-48ca-4fa2-91d8-fc6339b95743` + +Representative metadata: + +```text +run_id=20260405T213444Z +source_store=/frx/store/12d7704362e95afc2697db63f168b878e082b372-freebsd-source-default +guest_result_root=/var/lib/fruix/native-builds/20260405T213444Z +result_store=/frx/store/c6329a0053720b05aff3274b8b1d522c909f475d-fruix-native-build-result-15.0-STABLE-guest-self-hosted +world_store=/frx/store/dfe37b36f6537a95ceea16ea62001b2ca5617eb7-fruix-native-world-15.0-STABLE-guest-self-hosted +kernel_store=/frx/store/0ab7cbceca240ab2c3b91e83e059844ea792e49e-fruix-native-kernel-15.0-STABLE-guest-self-hosted +headers_store=/frx/store/5bbeae9266687a229f1c6d176a08886c35243ff0-fruix-native-headers-15.0-STABLE-guest-self-hosted +bootloader_store=/frx/store/bd49a508bd7a3b94a2535d6774f31c993c406552-fruix-native-bootloader-15.0-STABLE-guest-self-hosted +artifact_store_count=4 +sha_kernel=16950f116a52134b98e2f8e0dacc556e18fe254e4a0ac2c1741422dde281a341 +sha_loader=ea417846167ece270ada611624dca622ca38bd30125b9a125cd8ebb8b3600313 +sha_param=9eb140ca7d9666f3d484a4174c9acd94b45427db6292b4e17de19af2c6aa5219 +promoted_kernel_sha=16950f116a52134b98e2f8e0dacc556e18fe254e4a0ac2c1741422dde281a341 +promoted_loader_sha=ea417846167ece270ada611624dca622ca38bd30125b9a125cd8ebb8b3600313 +promoted_param_sha=9eb140ca7d9666f3d484a4174c9acd94b45427db6292b4e17de19af2c6aa5219 +native_build_store_promotion=ok +``` + +Validated facts: + +- guest self-hosted native-build results remain available under `/var/lib/fruix/native-builds/...` as mutable staging/results +- those staged results are now sufficient to promote first-class Fruix identities on the host +- promotion creates four immutable artifact store objects plus one immutable result-bundle store object +- promoted metadata retains executor/source/build-policy/provenance information explicitly +- promoted kernel, bootloader, and headers hashes match the already validated staged artifacts +- the promoted `world` artifact is now also preserved as a first-class Fruix store object instead of remaining only as an unpromoted staged tree + +## Important implementation note + +The first full promotion attempt exposed an incorrect assumption in the validation surface: + +- the promoted/staged `world` artifact should be checked at: + - `bin/sh` +- not at: + - `usr/bin/sh` + +That path expectation was corrected in: + +- helper validation +- emitted promotion metadata +- the end-to-end promotion harness + +## Result + +Fruix now has a validated native-build object model with a clear split: + +- mutable native-build result roots under `/var/lib/fruix/native-builds/...` +- immutable promoted identities under `/frx/store/...` + +That makes native FreeBSD base builds feel substantially more Fruix-native: + +- build outputs have explicit immutable identities +- metadata is Fruix-native rather than implied only by ad hoc directory layout +- executor/source/provenance/build-policy remain attached to the promoted result +- the staged result area and the real store identity are now intentionally distinct + +## Next direction + +This suggests the next product step is not merely “more self-hosting”. + +It is to generalize this result model so different execution modes can converge on the same promoted object story, for example: + +- host-initiated in-guest builds +- guest self-hosted builds +- future executor variants + +That would move Fruix closer to a shared executor model rather than treating each validation path as a one-off harness. \ No newline at end of file diff --git a/docs/system-deployment-workflow.md b/docs/system-deployment-workflow.md index f6b3aff..d9ef38d 100644 --- a/docs/system-deployment-workflow.md +++ b/docs/system-deployment-workflow.md @@ -278,10 +278,15 @@ That helper: 2. recovers the materialized FreeBSD source store from: - `/run/current-system/metadata/store-layout.scm` 3. runs the native FreeBSD build/install phases inside the guest -4. records results under: +4. records staged results under: - `/var/lib/fruix/native-builds/` - `/var/lib/fruix/native-builds/latest` -5. keeps the heavier object/stage work under: +5. emits promotion metadata for first-class artifact identities covering: + - `world` + - `kernel` + - `headers` + - `bootloader` +6. keeps the heavier object/stage work under: - `/var/tmp/fruix-self-hosted-native-builds/` Important current detail: @@ -294,6 +299,60 @@ So the validated Phase 20.3 answer is: - a controlled guest self-hosted base-build prototype now works - but the simpler default operator flow should still be the Phase 20.2 host-initiated in-guest path unless there is a specific reason to push the build loop farther into the guest +### Promoting native-build results into first-class Fruix store objects + +The guest-side result root is now explicitly a **staging/result area**, not the final immutable identity. + +Current validated flow: + +1. run the in-guest helper so the guest records a result under: + - `/var/lib/fruix/native-builds/` +2. copy that result root back to the host +3. run: + +```sh +fruix native-build promote RESULT_ROOT +``` + +The promotion step creates immutable `/frx/store` identities for: + +- `world` +- `kernel` +- `headers` +- `bootloader` + +and also creates a result-bundle store object that references those promoted artifact stores. + +Current metadata split: + +- mutable staging/result root: + - `/var/lib/fruix/native-builds/` +- immutable artifact stores: + - `/frx/store/...-fruix-native-world-...` + - `/frx/store/...-fruix-native-kernel-...` + - `/frx/store/...-fruix-native-headers-...` + - `/frx/store/...-fruix-native-bootloader-...` +- immutable result bundle: + - `/frx/store/...-fruix-native-build-result-...` + +The promoted store objects record explicit Fruix-native metadata including at least: + +- executor / executor-version +- run-id / guest-host-name +- closure path +- source store provenance +- build policy +- artifact kind +- required-file expectations +- recorded content signatures and hashes + +This is the current Fruix-native answer to the question: + +- where should mutable native-build state live? + - `/var/lib/fruix/native-builds/...` +- where should immutable native-build identity live? + - `/frx/store/...` + ## Deployment patterns ### 1. Build-first workflow diff --git a/modules/fruix/system/freebsd.scm b/modules/fruix/system/freebsd.scm index 35c9806..b4ad21b 100644 --- a/modules/fruix/system/freebsd.scm +++ b/modules/fruix/system/freebsd.scm @@ -1,6 +1,7 @@ (define-module (fruix system freebsd) #:use-module (fruix system freebsd model) #:use-module (fruix system freebsd source) + #:use-module (fruix system freebsd build) #:use-module (fruix system freebsd media) #:re-export (user-group user-group? @@ -43,6 +44,7 @@ operating-system-root-authorized-keys validate-operating-system materialize-freebsd-source + promote-native-build-result operating-system-closure-spec operating-system-install-spec operating-system-image-spec diff --git a/modules/fruix/system/freebsd/build.scm b/modules/fruix/system/freebsd/build.scm index 253ed8d..b9dba7b 100644 --- a/modules/fruix/system/freebsd/build.scm +++ b/modules/fruix/system/freebsd/build.scm @@ -11,6 +11,7 @@ #:use-module (srfi srfi-13) #:export (host-freebsd-provenance materialize-freebsd-package + promote-native-build-result materialize-prefix)) (define (host-freebsd-provenance) @@ -388,6 +389,182 @@ output-path)))) +(define native-build-result-promotion-version "1") + +(define (native-build-result-ref data key default) + (match (assoc key data) + ((_ . value) value) + (#f default))) + +(define (read-native-build-result result-root) + (let ((promotion-file (string-append result-root "/promotion.scm"))) + (unless (file-exists? promotion-file) + (error "native build result is missing promotion.scm" result-root)) + (let ((result (call-with-input-file promotion-file read))) + (unless (equal? (native-build-result-ref result 'native-build-result-version #f) + native-build-result-promotion-version) + (error "unsupported native build result promotion version" promotion-file)) + result))) + +(define (native-build-artifact-entry result artifact-kind) + (let* ((artifacts (native-build-result-ref result 'artifacts '())) + (entry (assoc artifact-kind artifacts))) + (unless entry + (error "native build result is missing artifact entry" artifact-kind)) + (cdr entry))) + +(define (native-build-artifact-root result-root result artifact-kind) + (let* ((entry (native-build-artifact-entry result artifact-kind)) + (relative-path (native-build-result-ref entry 'path #f)) + (required-file (native-build-result-ref entry 'required-file #f)) + (artifact-root (and relative-path + (string-append result-root "/" relative-path)))) + (unless (and artifact-root (file-exists? artifact-root)) + (error "native build result is missing artifact tree" artifact-kind artifact-root)) + (when required-file + (unless (file-exists? (string-append artifact-root "/" required-file)) + (error "native build artifact is missing required file" + artifact-kind + (string-append artifact-root "/" required-file)))) + artifact-root)) + +(define (native-build-existing-store-references result store-dir) + (filter identity + (map (lambda (path) + (and (string? path) + (string-prefix? (string-append store-dir "/") path) + (file-exists? path) + path)) + (list (native-build-result-ref result 'closure-path #f) + (let ((source (native-build-result-ref result 'source '()))) + (native-build-result-ref source 'store-path #f)))))) + +(define (native-build-artifact-display-name result artifact-kind) + (let* ((base (native-build-result-ref result 'freebsd-base '())) + (version-label (native-build-result-ref base 'version-label "unknown")) + (executor (native-build-result-ref result 'executor "unknown"))) + (string-append "fruix-native-" + (symbol->string artifact-kind) + "-" + version-label + "-" + executor))) + +(define (native-build-promoted-artifact-metadata result artifact-kind content-signature) + (let* ((entry (native-build-artifact-entry result artifact-kind))) + `((native-build-object-version . ,native-build-result-promotion-version) + (object-kind . artifact) + (artifact-kind . ,artifact-kind) + (executor . ,(native-build-result-ref result 'executor "unknown")) + (executor-version . ,(native-build-result-ref result 'executor-version "unknown")) + (run-id . ,(native-build-result-ref result 'run-id "unknown")) + (guest-host-name . ,(native-build-result-ref result 'guest-host-name "unknown")) + (closure-path . ,(native-build-result-ref result 'closure-path "")) + (development-profile . ,(native-build-result-ref result 'development-profile "")) + (freebsd-base . ,(native-build-result-ref result 'freebsd-base '())) + (source . ,(native-build-result-ref result 'source '())) + (build-policy . ,(native-build-result-ref result 'build-policy '())) + (required-file . ,(native-build-result-ref entry 'required-file "")) + (recorded-sha256 . ,(native-build-result-ref entry 'recorded-sha256 "")) + (content-signature . ,content-signature)))) + +(define (promote-native-build-artifact result-root result store-dir artifact-kind) + (let* ((artifact-root (native-build-artifact-root result-root result artifact-kind)) + (content-signature (tree-content-signature artifact-root)) + (metadata (native-build-promoted-artifact-metadata result artifact-kind content-signature)) + (payload (object->string metadata)) + (display-name (native-build-artifact-display-name result artifact-kind)) + (output-path (make-store-path store-dir display-name payload + #:kind 'native-build-artifact + #:output artifact-kind)) + (references (native-build-existing-store-references result store-dir))) + (unless (file-exists? output-path) + (mkdir-p output-path) + (stage-tree-into-output artifact-root output-path) + (write-file (string-append output-path "/.references") + (string-join references "\n")) + (write-file (string-append output-path "/.fruix-native-build-object.scm") + payload)) + `((artifact-kind . ,artifact-kind) + (artifact-root . ,artifact-root) + (store-path . ,output-path) + (content-signature . ,content-signature) + (metadata-file . ,(string-append output-path "/.fruix-native-build-object.scm")))) ) + +(define (native-build-result-display-name result) + (let* ((base (native-build-result-ref result 'freebsd-base '())) + (version-label (native-build-result-ref base 'version-label "unknown")) + (executor (native-build-result-ref result 'executor "unknown"))) + (string-append "fruix-native-build-result-" version-label "-" executor))) + +(define (native-build-promoted-result-object result promoted-artifacts) + `((native-build-result-version . ,native-build-result-promotion-version) + (object-kind . result-bundle) + (executor . ,(native-build-result-ref result 'executor "unknown")) + (executor-version . ,(native-build-result-ref result 'executor-version "unknown")) + (run-id . ,(native-build-result-ref result 'run-id "unknown")) + (guest-host-name . ,(native-build-result-ref result 'guest-host-name "unknown")) + (closure-path . ,(native-build-result-ref result 'closure-path "")) + (development-profile . ,(native-build-result-ref result 'development-profile "")) + (freebsd-base . ,(native-build-result-ref result 'freebsd-base '())) + (source . ,(native-build-result-ref result 'source '())) + (build-policy . ,(native-build-result-ref result 'build-policy '())) + (artifact-count . ,(length promoted-artifacts)) + (artifacts . ,(map (lambda (entry) + `((artifact-kind . ,(assoc-ref entry 'artifact-kind)) + (store-path . ,(assoc-ref entry 'store-path)) + (content-signature . ,(assoc-ref entry 'content-signature)) + (metadata-file . ,(assoc-ref entry 'metadata-file)))) + promoted-artifacts)))) + +(define* (promote-native-build-result result-root #:key (store-dir "/frx/store")) + (let* ((result (read-native-build-result result-root)) + (promoted-artifacts (map (lambda (artifact-kind) + (promote-native-build-artifact result-root result store-dir artifact-kind)) + '(world kernel headers bootloader))) + (result-object (native-build-promoted-result-object result promoted-artifacts)) + (payload (object->string result-object)) + (display-name (native-build-result-display-name result)) + (result-store (make-store-path store-dir display-name payload + #:kind 'native-build-result)) + (result-references (append (map (lambda (entry) + (assoc-ref entry 'store-path)) + promoted-artifacts) + (native-build-existing-store-references result store-dir)))) + (unless (file-exists? result-store) + (mkdir-p (string-append result-store "/artifacts")) + (for-each (lambda (entry) + (symlink (assoc-ref entry 'store-path) + (string-append result-store + "/artifacts/" + (symbol->string (assoc-ref entry 'artifact-kind))))) + promoted-artifacts) + (write-file (string-append result-store "/.references") + (string-join result-references "\n")) + (write-file (string-append result-store "/.fruix-native-build-result.scm") + payload)) + `((result-root . ,result-root) + (result-store . ,result-store) + (result-metadata-file . ,(string-append result-store "/.fruix-native-build-result.scm")) + (artifact-store-count . ,(length promoted-artifacts)) + (artifact-stores . ,(map (lambda (entry) (assoc-ref entry 'store-path)) promoted-artifacts)) + (world-store . ,(assoc-ref (find (lambda (entry) + (eq? (assoc-ref entry 'artifact-kind) 'world)) + promoted-artifacts) + 'store-path)) + (kernel-store . ,(assoc-ref (find (lambda (entry) + (eq? (assoc-ref entry 'artifact-kind) 'kernel)) + promoted-artifacts) + 'store-path)) + (headers-store . ,(assoc-ref (find (lambda (entry) + (eq? (assoc-ref entry 'artifact-kind) 'headers)) + promoted-artifacts) + 'store-path)) + (bootloader-store . ,(assoc-ref (find (lambda (entry) + (eq? (assoc-ref entry 'artifact-kind) 'bootloader)) + promoted-artifacts) + 'store-path))))) + (define (sanitize-materialized-prefix name output-path) (cond ((string=? name "fruix-guile-extra") diff --git a/modules/fruix/system/freebsd/model.scm b/modules/fruix/system/freebsd/model.scm index 1c15852..2333aad 100644 --- a/modules/fruix/system/freebsd/model.scm +++ b/modules/fruix/system/freebsd/model.scm @@ -334,7 +334,7 @@ (development-environment-helper-version . ,(if (null? (operating-system-development-packages os)) #f "1")) (self-hosted-native-build-helper-version - . ,(if (null? (operating-system-development-packages os)) #f "2")) + . ,(if (null? (operating-system-development-packages os)) #f "3")) (user-count . ,(length (operating-system-users os))) (users . ,(map user-account-name (operating-system-users os))) (group-count . ,(length (operating-system-groups os))) diff --git a/modules/fruix/system/freebsd/render.scm b/modules/fruix/system/freebsd/render.scm index 8d52eda..5f6f115 100644 --- a/modules/fruix/system/freebsd/render.scm +++ b/modules/fruix/system/freebsd/render.scm @@ -787,6 +787,11 @@ "EOF\n")) (define (render-self-hosted-native-build-script os) (let* ((base-spec (freebsd-base-spec (operating-system-freebsd-base os))) + (base-name (assoc-ref base-spec 'name)) + (version-label (assoc-ref base-spec 'version-label)) + (release (assoc-ref base-spec 'release)) + (branch (assoc-ref base-spec 'branch)) + (declared-source-root (assoc-ref base-spec 'source-root)) (target (assoc-ref base-spec 'target)) (target-arch (assoc-ref base-spec 'target-arch)) (kernconf (assoc-ref base-spec 'kernconf)) @@ -803,6 +808,7 @@ "set -eu\n" "umask 022\n" "profile=/run/current-system/development-profile\n" + "guest_host_name='" (operating-system-host-name os) "'\n" "[ -d \"$profile\" ] || {\n" " echo \"fruix-self-hosted-native-build: development profile is not available\" >&2\n" " exit 1\n" @@ -846,8 +852,10 @@ "logdir=$result_root/logs\n" "status_file=$result_root/status\n" "metadata_file=$result_root/metadata.txt\n" + "promotion_file=$result_root/promotion.scm\n" "world_stage=$build_root/stage-world\n" "kernel_stage=$build_root/stage-kernel\n" + "world_artifact=$result_root/artifacts/world\n" "headers_artifact=$result_root/artifacts/headers\n" "kernel_artifact=$result_root/artifacts/kernel\n" "bootloader_artifact=$result_root/artifacts/bootloader\n" @@ -877,13 +885,14 @@ " echo \"fruix-self-hosted-native-build: source root is missing: $source_root\" >&2\n" " exit 1\n" "}\n" - "mkdir -p \"$headers_artifact/usr\" \"$kernel_artifact/boot\" \"$bootloader_artifact/boot\"\n" + "mkdir -p \"$world_artifact\" \"$headers_artifact/usr\" \"$kernel_artifact/boot\" \"$bootloader_artifact/boot\"\n" "export MAKEOBJDIRPREFIX=\"$build_root/obj\"\n" "make -j\"$jobs\" -C \"$source_root\" " build-common " buildworld > \"$logdir/buildworld.log\" 2>&1\n" "make -j\"$jobs\" -C \"$source_root\" " build-common " buildkernel > \"$logdir/buildkernel.log\" 2>&1\n" "make -C \"$source_root\" " install-common " DESTDIR=\"$world_stage\" installworld > \"$logdir/installworld.log\" 2>&1\n" "make -C \"$source_root\" " install-common " DESTDIR=\"$world_stage\" distribution > \"$logdir/distribution.log\" 2>&1\n" "make -C \"$source_root\" " install-common " DESTDIR=\"$kernel_stage\" installkernel > \"$logdir/installkernel.log\" 2>&1\n" + "cp -a \"$world_stage/.\" \"$world_artifact/\"\n" "cp -a \"$kernel_stage/boot/kernel\" \"$kernel_artifact/boot/kernel\"\n" "cp -a \"$world_stage/usr/include\" \"$headers_artifact/usr/include\"\n" "mkdir -p \"$headers_artifact/usr/share\"\n" @@ -893,6 +902,7 @@ "cp -a \"$world_stage/boot/device.hints\" \"$bootloader_artifact/boot/device.hints\"\n" "cp -a \"$world_stage/boot/defaults\" \"$bootloader_artifact/boot/defaults\"\n" "cp -a \"$world_stage/boot/lua\" \"$bootloader_artifact/boot/lua\"\n" + "[ -f \"$world_artifact/bin/sh\" ]\n" "[ -f \"$kernel_artifact/boot/kernel/kernel\" ]\n" "[ -f \"$headers_artifact/usr/include/sys/param.h\" ]\n" "[ -f \"$headers_artifact/usr/share/mk/bsd.prog.mk\" ]\n" @@ -910,15 +920,50 @@ "root_df=$(df -h / | tail -n 1 | tr -s ' ' | tr '\\t' ' ')\n" "build_root_size=$(du -sh \"$build_root\" | awk '{print $1}')\n" "result_root_size=$(du -sh \"$result_root\" | awk '{print $1}')\n" + "world_artifact_size=$(du -sh \"$world_artifact\" | awk '{print $1}')\n" "kernel_artifact_size=$(du -sh \"$kernel_artifact\" | awk '{print $1}')\n" "headers_artifact_size=$(du -sh \"$headers_artifact\" | awk '{print $1}')\n" "bootloader_artifact_size=$(du -sh \"$bootloader_artifact\" | awk '{print $1}')\n" "rm -f \"$latest_link\"\n" "ln -s \"$result_root\" \"$latest_link\"\n" + "cat >\"$promotion_file\" <\"$metadata_file\" <string (stat:type st))))))) +(define (tree-content-signature root) + (define (walk path relative) + (let ((st (lstat path))) + (case (stat:type st) + ((regular) + (string-append "file:" relative ":" (file-hash path))) + ((symlink) + (string-append "symlink:" relative ":" (readlink path))) + ((directory) + (string-join + (cons (string-append "directory:" relative) + (apply append + (map (lambda (entry) + (let ((child-relative (if (string=? relative ".") + entry + (string-append relative "/" entry)))) + (list (walk (string-append path "/" entry) + child-relative)))) + (directory-entries path)))) + "\n")) + (else + (string-append "other:" relative ":" (symbol->string (stat:type st))))))) + (walk root ".")) + (define (install-plan-signature entry) (match entry (('file source target) diff --git a/scripts/fruix.scm b/scripts/fruix.scm index 1ff8ead..4d2c4aa 100644 --- a/scripts/fruix.scm +++ b/scripts/fruix.scm @@ -15,6 +15,7 @@ Commands:\n\ system ACTION ... Build or materialize Fruix system artifacts.\n\ source ACTION ... Fetch or snapshot declarative FreeBSD source inputs.\n\ + native-build ACTION ... Promote native build results into Fruix store objects.\n\ \n\ System actions:\n\ build Materialize the Fruix system closure in /frx/store.\n\ @@ -37,6 +38,12 @@ System options:\n\ Source actions:\n\ materialize Materialize a declared FreeBSD source tree in /frx/store.\n\ \n\ +Native-build actions:\n\ + promote Promote a native build result root into /frx/store.\n\ +\n\ +Native-build options:\n\ + --store DIR Store directory to use (default: /frx/store).\n\ +\n\ Source options:\n\ --source NAME Scheme variable holding the freebsd-source object.\n\ --store DIR Store directory to use (default: /frx/store).\n\ @@ -216,6 +223,28 @@ Common options:\n\ ((arg . tail) (loop tail (cons arg positional) source-name store-dir cache-dir))))) +(define (parse-native-build-arguments action rest) + (let loop ((args rest) + (positional '()) + (store-dir "/frx/store")) + (match args + (() + (let ((positional (reverse positional))) + `((command . "native-build") + (action . ,action) + (positional . ,positional) + (store-dir . ,store-dir)))) + (("--help") + (usage 0)) + (((? (lambda (arg) (string-prefix? "--store=" arg)) arg) . tail) + (loop tail positional (option-value arg "--store="))) + (("--store" value . tail) + (loop tail positional value)) + (((? (lambda (arg) (string-prefix? "--" arg)) arg) . _) + (error "unknown option" arg)) + ((arg . tail) + (loop tail (cons arg positional) store-dir))))) + (define (parse-arguments argv) (match argv ((_) @@ -228,10 +257,14 @@ Common options:\n\ (usage 0)) ((_ "source" "--help") (usage 0)) + ((_ "native-build" "--help") + (usage 0)) ((_ "system" action . rest) (parse-system-arguments action rest)) ((_ "source" action . rest) (parse-source-arguments action rest)) + ((_ "native-build" action . rest) + (parse-native-build-arguments action rest)) ((_ . _) (usage 1)))) @@ -542,6 +575,20 @@ Common options:\n\ (target_store_item_count . ,(length target-store-items)) (installer_store_item_count . ,(length installer-store-items)))))) +(define (emit-native-build-promotion-metadata store-dir result-root result) + (emit-metadata + `((action . "promote") + (result_root . ,result-root) + (store_dir . ,store-dir) + (result_store . ,(assoc-ref result 'result-store)) + (result_metadata_file . ,(assoc-ref result 'result-metadata-file)) + (artifact_store_count . ,(assoc-ref result 'artifact-store-count)) + (artifact_stores . ,(string-join (assoc-ref result 'artifact-stores) ",")) + (world_store . ,(assoc-ref result 'world-store)) + (kernel_store . ,(assoc-ref result 'kernel-store)) + (headers_store . ,(assoc-ref result 'headers-store)) + (bootloader_store . ,(assoc-ref result 'bootloader-store))))) + (define (main argv) (let* ((parsed (parse-arguments argv)) (command (assoc-ref parsed 'command)) @@ -692,6 +739,16 @@ Common options:\n\ (materialized_source_ref . ,(or (assoc-ref effective 'ref) "")) (materialized_source_commit . ,(or (assoc-ref result 'effective-commit) "")) (materialized_source_sha256 . ,(or (assoc-ref result 'effective-sha256) "")))))))))) + ((string=? command "native-build") + (let ((positional (assoc-ref parsed 'positional))) + (unless (string=? action "promote") + (error "unknown native-build action" action)) + (let ((result-root (match positional + ((path . _) path) + (() (error "missing native build result root argument"))))) + (emit-native-build-promotion-metadata + store-dir result-root + (promote-native-build-result result-root #:store-dir store-dir))))) (else (usage 1))))) diff --git a/tests/system/run-phase20-native-build-store-promotion-xcpng.sh b/tests/system/run-phase20-native-build-store-promotion-xcpng.sh new file mode 100755 index 0000000..7419aff --- /dev/null +++ b/tests/system/run-phase20-native-build-store-promotion-xcpng.sh @@ -0,0 +1,231 @@ +#!/bin/sh +set -eu + +repo_root=${PROJECT_ROOT:-$(pwd)} +os_template=${OS_TEMPLATE:-$repo_root/tests/system/phase20-development-operating-system.scm.in} +system_name=${SYSTEM_NAME:-phase20-operating-system} +root_size=${ROOT_SIZE:-20g} +store_dir=${STORE_DIR:-/frx/store} +metadata_target=${METADATA_OUT:-} +root_authorized_key_file=${ROOT_AUTHORIZED_KEY_FILE:-$HOME/.ssh/id_ed25519.pub} +root_ssh_private_key_file=${ROOT_SSH_PRIVATE_KEY_FILE:-$HOME/.ssh/id_ed25519} +cleanup=0 + +if [ -n "${WORKDIR:-}" ]; then + workdir=$WORKDIR + mkdir -p "$workdir" +else + workdir=$(mktemp -d /tmp/fruix-phase20-native-build-store-promotion-xcpng.XXXXXX) + cleanup=1 +fi +if [ "${KEEP_WORKDIR:-0}" -eq 1 ]; then + cleanup=0 +fi + +inner_metadata=$workdir/phase20-native-build-store-promotion-inner-metadata.txt +existing_inner_metadata=${EXISTING_INNER_METADATA:-} +promotion_out=$workdir/native-build-promote.txt +metadata_file=$workdir/phase20-native-build-store-promotion-xcpng-metadata.txt +import_root=$workdir/import + +action_cleanup() { + if [ "$cleanup" -eq 1 ]; then + rm -rf "$workdir" + fi +} +trap action_cleanup EXIT INT TERM + +if [ -n "$existing_inner_metadata" ]; then + cp "$existing_inner_metadata" "$inner_metadata" +else + KEEP_WORKDIR=1 WORKDIR="$workdir/inner" METADATA_OUT="$inner_metadata" \ + ROOT_AUTHORIZED_KEY_FILE="$root_authorized_key_file" \ + ROOT_SSH_PRIVATE_KEY_FILE="$root_ssh_private_key_file" \ + OS_TEMPLATE="$os_template" SYSTEM_NAME="$system_name" ROOT_SIZE="$root_size" \ + "$repo_root/tests/system/run-phase20-self-hosted-native-build-xcpng.sh" +fi + +guest_ip=$(sed -n 's/^guest_ip=//p' "$inner_metadata") +vm_id=$(sed -n 's/^vm_id=//p' "$inner_metadata") +vdi_id=$(sed -n 's/^vdi_id=//p' "$inner_metadata") +closure_path=$(sed -n 's/^closure_path=//p' "$inner_metadata") +closure_base=$(sed -n 's/^closure_base=//p' "$inner_metadata") +run_id=$(sed -n 's/^run_id=//p' "$inner_metadata") +source_store=$(sed -n 's/^source_store=//p' "$inner_metadata") +result_root=$(sed -n 's/^result_root=//p' "$inner_metadata") +promotion_file=$(sed -n 's/^promotion_file=//p' "$inner_metadata") +world_artifact=$(sed -n 's/^world_artifact=//p' "$inner_metadata") +kernel_artifact=$(sed -n 's/^kernel_artifact=//p' "$inner_metadata") +headers_artifact=$(sed -n 's/^headers_artifact=//p' "$inner_metadata") +bootloader_artifact=$(sed -n 's/^bootloader_artifact=//p' "$inner_metadata") +sha_kernel=$(sed -n 's/^sha_kernel=//p' "$inner_metadata") +sha_loader=$(sed -n 's/^sha_loader=//p' "$inner_metadata") +sha_param=$(sed -n 's/^sha_param=//p' "$inner_metadata") + +ssh_guest() { + ssh -i "$root_ssh_private_key_file" \ + -o BatchMode=yes \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + root@"$guest_ip" "$@" +} + +mkdir -p "$import_root" +result_base=$(basename "$result_root") +ssh -i "$root_ssh_private_key_file" \ + -o BatchMode=yes \ + -o StrictHostKeyChecking=no \ + -o UserKnownHostsFile=/dev/null \ + -o ConnectTimeout=5 \ + root@"$guest_ip" "tar -C '$(dirname "$result_root")' -cf - '$result_base'" | tar -C "$import_root" -xf - +local_result_root=$import_root/$result_base +[ -d "$local_result_root" ] || { echo "failed to import native build result root" >&2; exit 1; } +[ -f "$local_result_root/promotion.scm" ] || { echo "imported result is missing promotion.scm" >&2; exit 1; } +[ -f "$local_result_root/artifacts/world/bin/sh" ] || { echo "imported result is missing world artifact" >&2; exit 1; } +[ -f "$local_result_root/artifacts/kernel/boot/kernel/kernel" ] || { echo "imported result is missing kernel artifact" >&2; exit 1; } +[ -f "$local_result_root/artifacts/headers/usr/include/sys/param.h" ] || { echo "imported result is missing headers artifact" >&2; exit 1; } +[ -f "$local_result_root/artifacts/bootloader/boot/loader.efi" ] || { echo "imported result is missing bootloader artifact" >&2; exit 1; } + +action_env() { + sudo env \ + HOME="$HOME" \ + GUILE_AUTO_COMPILE=0 \ + GUIX_SOURCE_DIR="${GUIX_SOURCE_DIR:-$HOME/repos/guix}" \ + GUILE_BIN="${GUILE_BIN:-/tmp/guile-freebsd-validate-install/bin/guile}" \ + GUILE_EXTRA_PREFIX="${GUILE_EXTRA_PREFIX:-/tmp/guile-gnutls-freebsd-validate-install}" \ + SHEPHERD_PREFIX="${SHEPHERD_PREFIX:-/tmp/shepherd-freebsd-validate-install}" \ + "$@" +} + +action_env "$repo_root/bin/fruix" native-build promote "$local_result_root" --store "$store_dir" >"$promotion_out" + +field() { + sed -n "s/^$1=//p" "$promotion_out" | tail -n 1 +} + +result_store=$(field result_store) +result_metadata_file=$(field result_metadata_file) +artifact_store_count=$(field artifact_store_count) +artifact_stores=$(field artifact_stores) +world_store=$(field world_store) +kernel_store=$(field kernel_store) +headers_store=$(field headers_store) +bootloader_store=$(field bootloader_store) + +[ "$artifact_store_count" = 4 ] || { echo "unexpected artifact store count: $artifact_store_count" >&2; exit 1; } +case "$result_store" in + /frx/store/*-fruix-native-build-result-*-guest-self-hosted) : ;; + *) echo "unexpected result store path: $result_store" >&2; exit 1 ;; +esac +case "$world_store" in + /frx/store/*-fruix-native-world-*-guest-self-hosted) : ;; + *) echo "unexpected world store path: $world_store" >&2; exit 1 ;; +esac +case "$kernel_store" in + /frx/store/*-fruix-native-kernel-*-guest-self-hosted) : ;; + *) echo "unexpected kernel store path: $kernel_store" >&2; exit 1 ;; +esac +case "$headers_store" in + /frx/store/*-fruix-native-headers-*-guest-self-hosted) : ;; + *) echo "unexpected headers store path: $headers_store" >&2; exit 1 ;; +esac +case "$bootloader_store" in + /frx/store/*-fruix-native-bootloader-*-guest-self-hosted) : ;; + *) echo "unexpected bootloader store path: $bootloader_store" >&2; exit 1 ;; +esac + +[ -f "$result_metadata_file" ] || { echo "missing result metadata file: $result_metadata_file" >&2; exit 1; } +[ -f "$world_store/.fruix-native-build-object.scm" ] || { echo "missing world store metadata" >&2; exit 1; } +[ -f "$kernel_store/.fruix-native-build-object.scm" ] || { echo "missing kernel store metadata" >&2; exit 1; } +[ -f "$headers_store/.fruix-native-build-object.scm" ] || { echo "missing headers store metadata" >&2; exit 1; } +[ -f "$bootloader_store/.fruix-native-build-object.scm" ] || { echo "missing bootloader store metadata" >&2; exit 1; } +[ -L "$result_store/artifacts/world" ] || { echo "missing promoted world artifact link" >&2; exit 1; } +[ -L "$result_store/artifacts/kernel" ] || { echo "missing promoted kernel artifact link" >&2; exit 1; } +[ -L "$result_store/artifacts/headers" ] || { echo "missing promoted headers artifact link" >&2; exit 1; } +[ -L "$result_store/artifacts/bootloader" ] || { echo "missing promoted bootloader artifact link" >&2; exit 1; } +[ "$(readlink "$result_store/artifacts/world")" = "$world_store" ] || { echo "world artifact link mismatch" >&2; exit 1; } +[ "$(readlink "$result_store/artifacts/kernel")" = "$kernel_store" ] || { echo "kernel artifact link mismatch" >&2; exit 1; } +[ "$(readlink "$result_store/artifacts/headers")" = "$headers_store" ] || { echo "headers artifact link mismatch" >&2; exit 1; } +[ "$(readlink "$result_store/artifacts/bootloader")" = "$bootloader_store" ] || { echo "bootloader artifact link mismatch" >&2; exit 1; } +[ -f "$world_store/bin/sh" ] || { echo "promoted world store missing /bin/sh" >&2; exit 1; } +[ -f "$kernel_store/boot/kernel/kernel" ] || { echo "promoted kernel store missing kernel" >&2; exit 1; } +[ -f "$headers_store/usr/include/sys/param.h" ] || { echo "promoted headers store missing param.h" >&2; exit 1; } +[ -f "$bootloader_store/boot/loader.efi" ] || { echo "promoted bootloader store missing loader.efi" >&2; exit 1; } + +promoted_kernel_sha=$(sha256 -q "$kernel_store/boot/kernel/kernel") +promoted_loader_sha=$(sha256 -q "$bootloader_store/boot/loader.efi") +promoted_param_sha=$(sha256 -q "$headers_store/usr/include/sys/param.h") +[ "$promoted_kernel_sha" = "$sha_kernel" ] || { echo "kernel sha mismatch after promotion" >&2; exit 1; } +[ "$promoted_loader_sha" = "$sha_loader" ] || { echo "loader sha mismatch after promotion" >&2; exit 1; } +[ "$promoted_param_sha" = "$sha_param" ] || { echo "param.h sha mismatch after promotion" >&2; exit 1; } + +grep -F '(executor . "guest-self-hosted")' "$result_metadata_file" >/dev/null || { + echo "result metadata file is missing guest-self-hosted executor" >&2 + exit 1 +} +grep -F "$source_store" "$result_metadata_file" >/dev/null || { + echo "result metadata file is missing source store provenance" >&2 + exit 1 +} +grep -F '(artifact-kind . kernel)' "$kernel_store/.fruix-native-build-object.scm" >/dev/null || { + echo "kernel store metadata is missing artifact kind" >&2 + exit 1 +} +grep -F '(artifact-kind . world)' "$world_store/.fruix-native-build-object.scm" >/dev/null || { + echo "world store metadata is missing artifact kind" >&2 + exit 1 +} + +cat >"$metadata_file" <&2; exit 1; } +[ "$helper_version" = 3 ] || { echo "unexpected helper version: $helper_version" >&2; exit 1; } [ "$build_jobs" = "$guest_build_jobs" ] || { echo "unexpected build job count: $build_jobs" >&2; exit 1; } [ "$status_value" = ok ] || { echo "self-hosted build status is not ok: $status_value" >&2; exit 1; } [ "$latest_target" = "$result_root" ] || { echo "latest link target mismatch: $latest_target" >&2; exit 1; } @@ -202,8 +207,10 @@ result_root=$result_root logdir=$logdir status_file=$status_file guest_metadata_file=$guest_metadata_file +promotion_file=$promotion_file world_stage=$world_stage kernel_stage=$kernel_stage +world_artifact=$world_artifact kernel_artifact=$kernel_artifact headers_artifact=$headers_artifact bootloader_artifact=$bootloader_artifact @@ -213,6 +220,7 @@ status_value=$status_value root_df=$root_df build_root_size=$build_root_size result_root_size=$result_root_size +world_artifact_size=$world_artifact_size kernel_artifact_size=$kernel_artifact_size headers_artifact_size=$headers_artifact_size bootloader_artifact_size=$bootloader_artifact_size