13:48:17[2025-09-01T13:48:17.981Z] Handling message: {"artifact":{"type":"koji-build-group","builds":[{"type":"koji-build","id":2804120,"task_id":136272008,"nvr":"grub2-2.12-41.fc44"}]},"agent":"siosm","re-trigger":true,"update":{"autokarma":false,"autotime":true,"stable_karma":3,"stable_days":0,"unstable_karma":-3,"require_bugs":false,"require_testcases":false,"display_name":"","notes":"Automatic update for grub2-2.12-41.fc44.\n\n##### **Changelog**\n\n```\n* Mon Aug 11 2025 Marta Lewandowska <mlewando@redhat.com> - 2.12-41\n- Phase 1 of the bootloader updates proposal implementation\n- https://fedoraproject.org/wiki/Changes/BootLoaderUpdatesPhase1\n\n```","type":"unspecified","status":"testing","request":null,"severity":"unspecified","suggest":"unspecified","locked":false,"pushed":true,"critpath":true,"critpath_groups":"core critical-path-anaconda critical-path-base critical-path-compose critical-path-coreos","close_bugs":true,"date_submitted":"2025-08-20 16:51:22","date_modified":null,"date_approved":null,"date_testing":"2025-08-20 18:34:58","date_stable":null,"alias":"FEDORA-2025-e5c1f79161","test_gating_status":"failed","from_tag":null,"date_pushed":"2025-08-20 18:34:58","meets_testing_requirements":false,"url":"https://bodhi.fedoraproject.org/updates/FEDORA-2025-e5c1f79161","title":"grub2-2.12-41.fc44","version_hash":"a2c8367311a50fd40268305eaccb65463e04db00","release":{"name":"F44","long_name":"Fedora 44","version":"44","id_prefix":"FEDORA","branch":"rawhide","dist_tag":"f44","stable_tag":"f44","testing_tag":"f44-updates-testing","candidate_tag":"f44-updates-candidate","pending_signing_tag":"f44-signing-pending","pending_testing_tag":"f44-updates-testing-pending","pending_stable_tag":"f44-updates-pending","override_tag":"f44-override","mail_template":"fedora_errata_template","state":"pending","composed_by_bodhi":false,"create_automatic_updates":true,"package_manager":"unspecified","testing_repository":null,"released_on":null,"eol":null,"critpath_mandatory_days_in_testing":0,"mandatory_days_in_testing":0,"critpath_min_karma":0,"min_karma":0,"setting_status":"pre_beta"},"user":{"name":"lsandova","email":"lsandova@redhat.com","id":8326,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"fedorabugs"}]},"comments":[{"karma":0,"karma_critpath":0,"text":"This update was automatically created","timestamp":"2025-08-20 16:51:22","update_id":753030,"user_id":91,"id":4272738,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"bodhi","email":null,"id":91,"avatar":null,"openid":null,"groups":[]}},{"karma":0,"karma_critpath":0,"text":"This update's test gating status has been changed to 'waiting'.","timestamp":"2025-08-20 16:51:24","update_id":753030,"user_id":91,"id":4272739,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"bodhi","email":null,"id":91,"avatar":null,"openid":null,"groups":[]}},{"karma":0,"karma_critpath":0,"text":"This update's test gating status has been changed to 'failed'.","timestamp":"2025-08-20 16:59:55","update_id":753030,"user_id":91,"id":4272741,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"bodhi","email":null,"id":91,"avatar":null,"openid":null,"groups":[]}},{"karma":-1,"karma_critpath":0,"text":"This appears to break bootupd when building CoreOS images:\n\n```\n12:59:40 Failed to open file \"/sys/fs/selinux/checkreqprot\": Read-only file system\n12:59:40 error: boot data installation failed: Failed to find shimx64.efi in the image\n```","timestamp":"2025-08-20 17:02:34","update_id":753030,"user_id":659,"id":4272747,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"dustymabe","email":"dusty@dustymabe.com","id":659,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"sysadmin-badges"},{"name":"aws"},{"name":"qa-beaker-user"},{"name":"gitfedora-web"},{"name":"centosproject-email-aliases"},{"name":"fi-apprentice"},{"name":"sysadmin-veteran"},{"name":"web"},{"name":"sig-paas"},{"name":"atomic-wg"},{"name":"qa"},{"name":"fedorabugs"},{"name":"msftazure"},{"name":"pungi-devel"},{"name":"qa-automation-shell"},{"name":"sysadmin-fedimg"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"coreos"},{"name":"sysadmin"},{"name":"communishift"},{"name":"aws-fcos-mgmt"},{"name":"sysadmin-bodhi"},{"name":"sysadmin-releng"},{"name":"sysadmin-coreos"},{"name":"ipausers"},{"name":"ibmcloud"},{"name":"discussion-mods-project"},{"name":"discussion-mods-ask"},{"name":"hetznercloud"},{"name":"coreos-sig"}]}},{"karma":0,"karma_critpath":0,"text":"This update's test gating status has been changed to 'waiting'.","timestamp":"2025-08-20 18:34:58","update_id":753030,"user_id":91,"id":4272789,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"bodhi","email":null,"id":91,"avatar":null,"openid":null,"groups":[]}},{"karma":0,"karma_critpath":0,"text":"This update's test gating status has been changed to 'failed'.","timestamp":"2025-08-20 18:34:59","update_id":753030,"user_id":91,"id":4272790,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"bodhi","email":null,"id":91,"avatar":null,"openid":null,"groups":[]}},{"karma":-1,"karma_critpath":0,"text":"Yep, openQA hit just the same.\n```\n19:57:38,313 WARNING org.fedoraproject.Anaconda.Modules.Payloads:INFO:program:Running in chroot '/mnt/sysroot'... bootupctl backend install --auto --write-uuid --update-firmware --device /dev/vda /\n19:57:38,379 INFO systemd:mnt-sysimage-ostree-deploy-fedora-deploy-0cbe647bc8184f3d89df9a679ad154a90464155e355b3fa85ab5e25a9f063a56.0-boot-efi.mount: Deactivated successfully.\n19:57:38,379 INFO systemd:mnt-sysroot-boot-efi.mount: Deactivated successfully.\n19:57:38,383 WARNING org.fedoraproject.Anaconda.Modules.Payloads:INFO:program:error: boot data installation failed: installing component EFI: Failed to find shimx64.efi in the image\n19:57:38,383 WARNING org.fedoraproject.Anaconda.Modules.Payloads:DEBUG:program:Return code of bootupctl: 1\n```","timestamp":"2025-08-21 21:50:22","update_id":753030,"user_id":302,"id":4274109,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"adamwill","email":"awilliam@redhat.com","id":302,"avatar":null,"openid":null,"groups":[{"name":"proventesters"},{"name":"provenpackager"},{"name":"packager"},{"name":"qa-tools-sig"},{"name":"fedora-contributor"},{"name":"news"},{"name":"gittriage"},{"name":"sysadmin"},{"name":"gitspin-kickstarts"},{"name":"signed_fpca"},{"name":"gitgeneric-release"},{"name":"yak_farmers"},{"name":"triagers"},{"name":"qa"},{"name":"sysadmin-qa"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"gitmkinitrd"},{"name":"qa-admin"},{"name":"server-wg"},{"name":"forge-cle-members"},{"name":"aws-qa"},{"name":"change-wranglers"},{"name":"fedora-ci-admins"},{"name":"common-issues-triage"},{"name":"sysadmin-main"},{"name":"program-management"},{"name":"gitfedora-project-schedule"}]}},{"karma":0,"karma_critpath":0,"text":"Uh, sorry - openQA hits that error during install of Silverblue on UEFI. BIOS install works.","timestamp":"2025-08-21 21:51:36","update_id":753030,"user_id":302,"id":4274110,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"adamwill","email":"awilliam@redhat.com","id":302,"avatar":null,"openid":null,"groups":[{"name":"proventesters"},{"name":"provenpackager"},{"name":"packager"},{"name":"qa-tools-sig"},{"name":"fedora-contributor"},{"name":"news"},{"name":"gittriage"},{"name":"sysadmin"},{"name":"gitspin-kickstarts"},{"name":"signed_fpca"},{"name":"gitgeneric-release"},{"name":"yak_farmers"},{"name":"triagers"},{"name":"qa"},{"name":"sysadmin-qa"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"gitmkinitrd"},{"name":"qa-admin"},{"name":"server-wg"},{"name":"forge-cle-members"},{"name":"aws-qa"},{"name":"change-wranglers"},{"name":"fedora-ci-admins"},{"name":"common-issues-triage"},{"name":"sysadmin-main"},{"name":"program-management"},{"name":"gitfedora-project-schedule"}]}},{"karma":0,"karma_critpath":0,"text":"@hhei you did lots of testing for this change but apparently there is something else needed from CoreOS. Can you please help debug this one?","timestamp":"2025-08-21 22:41:26","update_id":753030,"user_id":8326,"id":4274128,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"lsandova","email":"lsandova@redhat.com","id":8326,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"fedorabugs"}]}},{"karma":0,"karma_critpath":0,"text":"It's not just CoreOS - I expect we'd hit this problem on any UEFI immutable (aka Atomic, aka image mode) case that uses bootupd. openQA hits it installing Silverblue, as I mentioned above. I don't recall if IoT adopted bootupd yet, but if it did, I'd expect it to be affected too.","timestamp":"2025-08-21 22:51:21","update_id":753030,"user_id":302,"id":4274150,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"adamwill","email":"awilliam@redhat.com","id":302,"avatar":null,"openid":null,"groups":[{"name":"proventesters"},{"name":"provenpackager"},{"name":"packager"},{"name":"qa-tools-sig"},{"name":"fedora-contributor"},{"name":"news"},{"name":"gittriage"},{"name":"sysadmin"},{"name":"gitspin-kickstarts"},{"name":"signed_fpca"},{"name":"gitgeneric-release"},{"name":"yak_farmers"},{"name":"triagers"},{"name":"qa"},{"name":"sysadmin-qa"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"gitmkinitrd"},{"name":"qa-admin"},{"name":"server-wg"},{"name":"forge-cle-members"},{"name":"aws-qa"},{"name":"change-wranglers"},{"name":"fedora-ci-admins"},{"name":"common-issues-triage"},{"name":"sysadmin-main"},{"name":"program-management"},{"name":"gitfedora-project-schedule"}]}},{"karma":0,"karma_critpath":0,"text":"@lsandova and I at least traced it back to the `get_efi_vendor` function in `src/efi.rs` - that's where the error comes from. That function is looking for a shim file and taking the name of the directory it finds it in as the \"vendor\" name - but in this case it's not finding a shim file and bailing out. We can't tell *which* invocation of it (there are two in `efi.rs` and one in `bootupd.rs`) we're hitting. We think it's looking in `usr/lib/bootupd/updates/EFI` within the 'sysroot' it's passed - that's \"the payload directory for an available update for a component\", where the \"component\" is the EFI one. But we've no idea how that's supposed to get populated.\n\nThe big change in grub here is https://src.fedoraproject.org/rpms/grub2/c/e02b6e88c71642ecc063ae7aefbd09b9be80d50e?branch=rawhide , which - AIUI - basically makes grub put EFI stuff in `grub_efi_dir` (which is under `libdir`) instead of `efi_esp_dir` (which is /boot/efi/EFI/fedora , i.e. it's directly on the ESP). There's then a %posttrans script for the `-efi` subpackages which copies the whole of `grub_efi_dir` to `efi_esp_dir`, but this is *not* done on atomic/image mode, it's skipped if `/run/ostree-booted` exists, with the comment \"On image mode, bootupd takes care of installing bootloader updates to the ESP\".\n\nSo if bootupd is expecting the ESP to be populated here - if the `usr/lib/bootupd/updates/EFI` thing is populated from it, or whatever - each time `get_efi_vendor` is called, that expectation may have just broken. We may be in a bit of a catch-22 here, where we're expecting bootupd to do the work of populating the ESP, but bootupd fails if the ESP isn't populated?","timestamp":"2025-08-21 23:03:41","update_id":753030,"user_id":302,"id":4274156,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"adamwill","email":"awilliam@redhat.com","id":302,"avatar":null,"openid":null,"groups":[{"name":"proventesters"},{"name":"provenpackager"},{"name":"packager"},{"name":"qa-tools-sig"},{"name":"fedora-contributor"},{"name":"news"},{"name":"gittriage"},{"name":"sysadmin"},{"name":"gitspin-kickstarts"},{"name":"signed_fpca"},{"name":"gitgeneric-release"},{"name":"yak_farmers"},{"name":"triagers"},{"name":"qa"},{"name":"sysadmin-qa"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"gitmkinitrd"},{"name":"qa-admin"},{"name":"server-wg"},{"name":"forge-cle-members"},{"name":"aws-qa"},{"name":"change-wranglers"},{"name":"fedora-ci-admins"},{"name":"common-issues-triage"},{"name":"sysadmin-main"},{"name":"program-management"},{"name":"gitfedora-project-schedule"}]}},{"karma":0,"karma_critpath":0,"text":"Yes, seems I missed some testing, will checking now.","timestamp":"2025-08-25 01:38:51","update_id":753030,"user_id":7350,"id":4276537,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"hhei","email":"hhei@redhat.com","id":7350,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"fedorabugs"},{"name":"coreos-sig"}]}},{"karma":0,"karma_critpath":0,"text":"Find the root cause:\nI used the updated grub2 & shim ( the scratch build based on https://src.fedoraproject.org/rpms/shim/pull-request/5 ) in my testing, but if we only have the updated grub2, then shim files will be missed when doing `generate-update-metadata` as will copy files from `usr/lib/efi` (but not find shim files), see https://jenkins-coreos-ci.apps.ocp.fedoraproject.org/job/test-override/166\n```\nshim-x64-15.8-3.x86_64 (fedora-coreos-pool)\n...\n+ /usr/bin/bootupctl backend generate-update-metadata\nGenerated update layout for BIOS: grub2-tools-1:2.12-41.fc44.x86_64\nGenerated update layout for EFI: grub2-1:2.12-41.fc44\n```\n\nIn this case, maybe should add checking in bootupd, if shim or grub2 is not ready, then fall back to copy files from legacy `usr/lib/ostree-boot`.","timestamp":"2025-08-25 02:16:17","update_id":753030,"user_id":7350,"id":4276575,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"hhei","email":"hhei@redhat.com","id":7350,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"fedorabugs"},{"name":"coreos-sig"}]}},{"karma":0,"karma_critpath":0,"text":"If the fix is going to be in bootupd, then we can re-run the tests on this when the fixed bootupd is stable, and if the fix works, this will pass and go through.","timestamp":"2025-08-25 14:48:10","update_id":753030,"user_id":302,"id":4277213,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"adamwill","email":"awilliam@redhat.com","id":302,"avatar":null,"openid":null,"groups":[{"name":"proventesters"},{"name":"provenpackager"},{"name":"packager"},{"name":"qa-tools-sig"},{"name":"fedora-contributor"},{"name":"news"},{"name":"gittriage"},{"name":"sysadmin"},{"name":"gitspin-kickstarts"},{"name":"signed_fpca"},{"name":"gitgeneric-release"},{"name":"yak_farmers"},{"name":"triagers"},{"name":"qa"},{"name":"sysadmin-qa"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"gitmkinitrd"},{"name":"qa-admin"},{"name":"server-wg"},{"name":"forge-cle-members"},{"name":"aws-qa"},{"name":"change-wranglers"},{"name":"fedora-ci-admins"},{"name":"common-issues-triage"},{"name":"sysadmin-main"},{"name":"program-management"},{"name":"gitfedora-project-schedule"}]}},{"karma":0,"karma_critpath":0,"text":"@hhei thanks for the analysis and the proposed fix.\n\n@adamwill I have launched a shim build so both packages, shim and grub, would have the same upgrade logic and with the change that @hhei is proposing, testing would go fine this time.","timestamp":"2025-08-25 15:05:34","update_id":753030,"user_id":8326,"id":4277243,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"lsandova","email":"lsandova@redhat.com","id":8326,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"fedorabugs"}]}},{"karma":0,"karma_critpath":0,"text":"@dustymabe @adamwill I just need to do a minor change on shim (see https://bodhi.fedoraproject.org/updates/FEDORA-2025-6a839fa19e#comment-4280011) before you can re-run all tests. I let you know once shim fix is merged.","timestamp":"2025-08-27 14:42:33","update_id":753030,"user_id":8326,"id":4280040,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"lsandova","email":"lsandova@redhat.com","id":8326,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"fedorabugs"}]}},{"karma":1,"karma_critpath":0,"text":"now that the new shim is in this passes CoreOS tests","timestamp":"2025-08-28 13:57:30","update_id":753030,"user_id":659,"id":4281786,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"dustymabe","email":"dusty@dustymabe.com","id":659,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"sysadmin-badges"},{"name":"aws"},{"name":"qa-beaker-user"},{"name":"gitfedora-web"},{"name":"centosproject-email-aliases"},{"name":"fi-apprentice"},{"name":"sysadmin-veteran"},{"name":"web"},{"name":"sig-paas"},{"name":"atomic-wg"},{"name":"qa"},{"name":"fedorabugs"},{"name":"msftazure"},{"name":"pungi-devel"},{"name":"qa-automation-shell"},{"name":"sysadmin-fedimg"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"coreos"},{"name":"sysadmin"},{"name":"communishift"},{"name":"aws-fcos-mgmt"},{"name":"sysadmin-bodhi"},{"name":"sysadmin-releng"},{"name":"sysadmin-coreos"},{"name":"ipausers"},{"name":"ibmcloud"},{"name":"discussion-mods-project"},{"name":"discussion-mods-ask"},{"name":"hetznercloud"},{"name":"coreos-sig"}]}},{"karma":0,"karma_critpath":0,"text":"Now that shim landed we need this one as well for Bootable Containers systems: https://github.com/fedora-silverblue/issue-tracker/issues/671. I'm testing it.","timestamp":"2025-09-01 13:28:14","update_id":753030,"user_id":4567,"id":4285402,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"siosm","email":"travier@redhat.com","id":4567,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"kde-sig"},{"name":"coreos"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"sysadmin-troubleshoot"},{"name":"flatpak-sig"},{"name":"okd"},{"name":"sig-cloud"},{"name":"coreos-sig"},{"name":"metrics-sig"},{"name":"centosproject-email-aliases"},{"name":"hetznercloud"},{"name":"provenpackager"}]}},{"karma":1,"karma_critpath":0,"text":"Let's push this one now that the shim update landed.","timestamp":"2025-09-01 13:48:09","update_id":753030,"user_id":4567,"id":4285426,"bug_feedback":[],"testcase_feedback":[],"user":{"name":"siosm","email":"travier@redhat.com","id":4567,"avatar":null,"openid":null,"groups":[{"name":"packager"},{"name":"kde-sig"},{"name":"coreos"},{"name":"fedorabugs"},{"name":"ipausers"},{"name":"fedora-contributor"},{"name":"signed_fpca"},{"name":"sysadmin-troubleshoot"},{"name":"flatpak-sig"},{"name":"okd"},{"name":"sig-cloud"},{"name":"coreos-sig"},{"name":"metrics-sig"},{"name":"centosproject-email-aliases"},{"name":"hetznercloud"},{"name":"provenpackager"}]}}],"builds":[{"nvr":"grub2-2.12-41.fc44","signed":true,"release_id":93,"type":"rpm","epoch":1}],"bugs":[],"updateid":"FEDORA-2025-e5c1f79161","karma":1,"content_type":"rpm","test_cases":[]}}
![[Jenkins]](/static/56f30bbe/images/svgs/logo.svg){kind=logo}
