{"update": {"autokarma": true, "autotime": true, "stable_karma": 3, "stable_days": 14, "unstable_karma": -3, "require_bugs": false, "require_testcases": true, "display_name": "", "notes": "This update reverts problematic ELF destructor ordering changes (RHBZ#2239304) and replaces it with an alternative approach that fixes different application compatibility issues (RHBZ#2244992).", "type": "unspecified", "status": "stable", "request": null, "severity": "unspecified", "suggest": "unspecified", "locked": false, "pushed": true, "critpath": true, "critpath_groups": "core critical-path-anaconda critical-path-apps critical-path-base critical-path-build critical-path-compose critical-path-deepin-desktop critical-path-gnome critical-path-kde critical-path-lxde critical-path-lxqt critical-path-server critical-path-standard critical-path-xfce", "close_bugs": false, "date_submitted": "2023-10-20 09:55:07", "date_modified": null, "date_approved": "2023-10-21 21:27:13", "date_testing": "2023-10-21 02:39:21", "date_stable": "2023-10-23 02:58:05", "alias": "FEDORA-2023-7dfa1f6429", "test_gating_status": "passed", "from_tag": null, "date_pushed": "2023-10-23 02:58:05", "meets_testing_requirements": true, "url": "https://bodhi.fedoraproject.org/updates/FEDORA-2023-7dfa1f6429", "title": "glibc-2.37-12.fc38", "version_hash": "c2b303bc4b17da7465dfe0d9ee14470e11943878", "release": {"name": "F38", "long_name": "Fedora 38", "version": "38", "id_prefix": "FEDORA", "branch": "f38", "dist_tag": "f38", "stable_tag": "f38-updates", "testing_tag": "f38-updates-testing", "candidate_tag": "f38-updates-candidate", "pending_signing_tag": "f38-signing-pending", "pending_testing_tag": "f38-updates-testing-pending", "pending_stable_tag": "f38-updates-pending", "override_tag": "f38-override", "mail_template": "fedora_errata_template", "state": "archived", "composed_by_bodhi": true, "create_automatic_updates": false, "package_manager": "dnf", "testing_repository": "updates-testing", "released_on": null, "eol": "2024-05-21", "critpath_mandatory_days_in_testing": 14, "mandatory_days_in_testing": 7, "critpath_min_karma": 2, "min_karma": 1, "setting_status": null}, "user": {"name": "fweimer", "email": "fweimer@redhat.com", "id": 1825, "avatar": "https://seccdn.libravatar.org/avatar/24e99cbd8f3721a17417169347326bfdeabd903e02f4733f62cf86e708dad8e2?s=24&d=retro", "openid": "fweimer.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "provenpackager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "centosproject-email-aliases"}, {"name": "gitsecure-coding"}, {"name": "sig-isa"}, {"name": "gitlab-centos-sig-isa"}]}, "comments": [{"karma": 0, "karma_critpath": 0, "text": "This update has been submitted for testing by fweimer. ", "timestamp": "2023-10-20 09:55:07", "update_id": 558726, "user_id": 91, "id": 3252657, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 0, "karma_critpath": 0, "text": "This update's test gating status has been changed to 'waiting'.", "timestamp": "2023-10-20 09:55:09", "update_id": 558726, "user_id": 91, "id": 3252658, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 0, "karma_critpath": 0, "text": "This update's test gating status has been changed to 'passed'.", "timestamp": "2023-10-20 11:19:57", "update_id": 558726, "user_id": 91, "id": 3252738, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 1, "karma_critpath": 0, "text": "Can confirm cisco secureclient works as expected.", "timestamp": "2023-10-20 23:01:23", "update_id": 558726, "user_id": 4150, "id": 3253300, "bug_feedback": [{"karma": 1, "comment_id": 3253300, "bug_id": 2239304, "bug": {"bug_id": 2239304, "title": "glibc: Revert change to run ELF destructor in reverse constructor order", "security": false, "parent": false}}, {"karma": 1, "comment_id": 3253300, "bug_id": 2244992, "bug": {"bug_id": 2244992, "title": "glibc: Improve compatibility of ELF destructor ordering", "security": false, "parent": false}}], "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}, {"karma": 0, "karma_critpath": 0, "text": "This update's test gating status has been changed to 'failed'.", "timestamp": "2023-10-21 02:40:15", "update_id": 558726, "user_id": 91, "id": 3253561, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 0, "karma_critpath": 0, "text": "This update has been pushed to testing.", "timestamp": "2023-10-21 02:40:21", "update_id": 558726, "user_id": 91, "id": 3253571, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 0, "karma_critpath": 0, "text": "This update's test gating status has been changed to 'waiting'.", "timestamp": "2023-10-21 03:00:38", "update_id": 558726, "user_id": 91, "id": 3253680, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 0, "karma_critpath": 0, "text": "This update's test gating status has been changed to 'passed'.", "timestamp": "2023-10-21 08:19:39", "update_id": 558726, "user_id": 91, "id": 3253750, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 1, "karma_critpath": 0, "text": "Works.", "timestamp": "2023-10-21 10:19:04", "update_id": 558726, "user_id": 198, "id": 3253792, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bojan", "email": "bojan@rexursive.com", "id": 198, "avatar": "https://seccdn.libravatar.org/avatar/adb4c3f28bd6969871988f992b1bf71e404f03e3570fbfa0fc9bd7d0bf0003f8?s=24&d=retro", "openid": "bojan.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}]}}, {"karma": 0, "karma_critpath": 0, "text": "This update can be pushed to stable now if the maintainer wishes", "timestamp": "2023-10-21 10:19:08", "update_id": 558726, "user_id": 91, "id": 3253794, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 1, "karma_critpath": 0, "text": "Works for me too, thanks very much.", "timestamp": "2023-10-21 20:57:47", "update_id": 558726, "user_id": 7920, "id": 3254059, "bug_feedback": [{"karma": 0, "comment_id": 3254059, "bug_id": 2239304, "bug": {"bug_id": 2239304, "title": "glibc: Revert change to run ELF destructor in reverse constructor order", "security": false, "parent": false}}, {"karma": 0, "comment_id": 3254059, "bug_id": 2244992, "bug": {"bug_id": 2244992, "title": "glibc: Improve compatibility of ELF destructor ordering", "security": false, "parent": false}}], "testcase_feedback": [], "user": {"name": "daithiociagain", "email": "dksw.daithi@gmail.com", "id": 7920, "avatar": "https://seccdn.libravatar.org/avatar/7f1bedc2ba234e66f13d026687118bdc66e7af1cce559f480686ea3cc2b34586?s=24&d=retro", "openid": "daithiociagain.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}, {"karma": 0, "karma_critpath": 0, "text": "This update has been submitted for stable by bodhi. ", "timestamp": "2023-10-21 20:57:47", "update_id": 558726, "user_id": 91, "id": 3254060, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}, {"karma": 1, "karma_critpath": 0, "text": "Works great!  LGTM!  =)", "timestamp": "2023-10-21 21:27:13", "update_id": 558726, "user_id": 206, "id": 3254064, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "besser82", "email": "besser82.fpo@gmail.com", "id": 206, "avatar": "https://seccdn.libravatar.org/avatar/28bacfa3be75032390ac2c7a34599aeb825c3b62b2c55ef64821d2fc023b56cb?s=24&d=retro", "openid": "besser82.id.fedoraproject.org", "groups": [{"name": "proventesters"}, {"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "ambassadors"}, {"name": "designteam"}, {"name": "shogun-ca"}, {"name": "alt-gtk-de-sig"}, {"name": "trust admins"}]}}, {"karma": 1, "karma_critpath": 0, "text": "Also worked for me on F38. I did not test F37 or F39.", "timestamp": "2023-10-23 02:45:42", "update_id": 558726, "user_id": 7924, "id": 3254810, "bug_feedback": [{"karma": 0, "comment_id": 3254810, "bug_id": 2239304, "bug": {"bug_id": 2239304, "title": "glibc: Revert change to run ELF destructor in reverse constructor order", "security": false, "parent": false}}, {"karma": 0, "comment_id": 3254810, "bug_id": 2244992, "bug": {"bug_id": 2244992, "title": "glibc: Improve compatibility of ELF destructor ordering", "security": false, "parent": false}}], "testcase_feedback": [], "user": {"name": "jimstaffer", "email": "jim@yosj.com.au", "id": 7924, "avatar": "https://seccdn.libravatar.org/avatar/c77b57c0c59746534b7e151efee07e6327325e3e83d76a900ef6507ac63fe5ca?s=24&d=retro", "openid": "jimstaffer.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}, {"karma": 0, "karma_critpath": 0, "text": "This update has been pushed to stable.", "timestamp": "2023-10-23 02:59:07", "update_id": 558726, "user_id": 91, "id": 3254860, "bug_feedback": [], "testcase_feedback": [], "user": {"name": "bodhi", "email": null, "id": 91, "avatar": "https://apps.fedoraproject.org/img/icons/bodhi-24.png", "openid": "bodhi.id.fedoraproject.org", "groups": []}}], "builds": [{"nvr": "glibc-2.37-12.fc38", "signed": true, "release_id": 66, "type": "rpm", "epoch": 0}], "bugs": [{"bug_id": 2239304, "title": "glibc: Revert change to run ELF destructor in reverse constructor order", "security": false, "parent": false, "feedback": [{"karma": 0, "comment_id": 3205838, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "@fweimer", "timestamp": "2023-09-19 19:45:34", "update_id": 549520, "user_id": 302, "id": 3205838, "testcase_feedback": [], "user": {"name": "adamwill", "email": "awilliam@redhat.com", "id": 302, "avatar": "https://seccdn.libravatar.org/avatar/51720fbb2ec5dfd2bc005ac374a3bfd3190cb8f45f3bb8d6bdf35fb6a051d03b?s=24&d=retro", "openid": "adamwill.id.fedoraproject.org", "groups": [{"name": "proventesters"}, {"name": "server-wg"}, {"name": "provenpackager"}, {"name": "packager"}, {"name": "qa-tools-sig"}, {"name": "fedora-contributor"}, {"name": "news"}, {"name": "gittriage"}, {"name": "sysadmin"}, {"name": "gitspin-kickstarts"}, {"name": "signed_fpca"}, {"name": "gitgeneric-release"}, {"name": "yak_farmers"}, {"name": "triagers"}, {"name": "qa"}, {"name": "sysadmin-qa"}, {"name": "fedorabugs"}, {"name": "ipausers"}, {"name": "gitmkinitrd"}, {"name": "qa-admin"}, {"name": "aws-qa"}, {"name": "change-wranglers"}, {"name": "fedora-ci-admins"}, {"name": "common-issues-triage"}, {"name": "sysadmin-main"}, {"name": "program-management"}, {"name": "gitfedora-project-schedule"}]}}}, {"karma": 0, "comment_id": 3206832, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "@adamwill How can we reproduce this in an environment where we can run the failing process under a debugger, or with certain environment variables configured? Thanks.", "timestamp": "2023-09-20 04:11:09", "update_id": 549520, "user_id": 1825, "id": 3206832, "testcase_feedback": [], "user": {"name": "fweimer", "email": "fweimer@redhat.com", "id": 1825, "avatar": "https://seccdn.libravatar.org/avatar/24e99cbd8f3721a17417169347326bfdeabd903e02f4733f62cf86e708dad8e2?s=24&d=retro", "openid": "fweimer.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "provenpackager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "centosproject-email-aliases"}, {"name": "gitsecure-coding"}, {"name": "sig-isa"}, {"name": "gitlab-centos-sig-isa"}]}}}, {"karma": 1, "comment_id": 3215764, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "OpenQA tests for FreeIPA succeeded. The upgrade test showed known SELinux AVCs which are being taken care of already.", "timestamp": "2023-09-27 08:32:44", "update_id": 549520, "user_id": 1242, "id": 3215764, "testcase_feedback": [], "user": {"name": "abbra", "email": "abokovoy@redhat.com", "id": 1242, "avatar": "https://seccdn.libravatar.org/avatar/3f3c51cb81fb43e685e65ff696d895ac268ff71949128c02b81dec65a4405a0a?s=24&d=retro", "openid": "abbra.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "server-wg"}, {"name": "bind-dyndb-ldap"}, {"name": "gitfreeipa"}, {"name": "gitslapi-nis"}, {"name": "gitfreeipa-docs"}]}}}, {"karma": 0, "comment_id": 3211705, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "This backtrace is more interesting:\n\n```\nStack trace of thread 1959:\n#0  0x00007f81c8ab0884 __pthread_kill_implementation (libc.so.6 + 0x8e884)\n#1  0x00007f81c8a5fafe raise (libc.so.6 + 0x3dafe)\n#2  0x00007f81c8a4887f abort (libc.so.6 + 0x2687f)\n#3  0x00007f81c8a4879b __assert_fail_base.cold (libc.so.6 + 0x2679b)\n#4  0x00007f81c8a58187 __assert_fail (libc.so.6 + 0x36187)\n#5  0x00007f81c9030323 krb5int_key_delete (libkrb5support.so.0 + 0x6323)\n#6  0x00007f81c86f0e8b gssint_mechglue_fini (libgssapi_krb5.so.2 + 0xee8b)\n#7  0x00007f81c91f50f2 _dl_call_fini (ld-linux-x86-64.so.2 + 0x10f2)\n#8  0x00007f81c91f8e5e _dl_fini (ld-linux-x86-64.so.2 + 0x4e5e)\n#9  0x00007f81c8a621e6 __run_exit_handlers (libc.so.6 + 0x401e6)\n#10 0x00007f81c8a6232e exit (libc.so.6 + 0x4032e)\n#11 0x00005622a14583bc main (ipa-getkeytab + 0x63bc)\n#12 0x00007f81c8a49b8a __libc_start_call_main (libc.so.6 + 0x27b8a)\n#13 0x00007f81c8a49c4b __libc_start_main@@GLIBC_2.34 (libc.so.6 + 0x27c4b)\n#14 0x00005622a1459bb5 _start (ipa-getkeytab + 0x7bb5)\n```\n\nIt has `_dl_fini` in it, so it's very likely it's caused by the changes in this update.\n\n@adamwill @martinpitt How can I create a VM (or set of VMs) that reproduces this issue? Thanks.\n", "timestamp": "2023-09-24 06:45:51", "update_id": 549520, "user_id": 1825, "id": 3211705, "testcase_feedback": [], "user": {"name": "fweimer", "email": "fweimer@redhat.com", "id": 1825, "avatar": "https://seccdn.libravatar.org/avatar/24e99cbd8f3721a17417169347326bfdeabd903e02f4733f62cf86e708dad8e2?s=24&d=retro", "openid": "fweimer.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "provenpackager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "centosproject-email-aliases"}, {"name": "gitsecure-coding"}, {"name": "sig-isa"}, {"name": "gitlab-centos-sig-isa"}]}}}, {"karma": 0, "comment_id": 3211843, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "@fweimer this seems like unloading GSSAPI mechglue plugin after `ipa-getkeytab` successfully completed. \n\nJudging by cockpit logs:\n```\n/usr/libexec/cockpit-certificate-helper: line 86:  2147 Aborted                 (core dumped) ipa-getkeytab -p \"HTTP/${HOST}\" -k \"${KEYTAB}\"\n```\n\nthis is, again, a very basic 'ipa-getkeytab' operation that attempts to store/delete a key in the keytab: a single process, single thread operation, nothing fancy. I think this is the code corresponding to `krb5int_key_delete` (there are macro definitions that bring it from `k5_key_delete` name): https://github.com/krb5/krb5/blob/master/src/util/support/threads.c#L362-L397", "timestamp": "2023-09-24 11:15:16", "update_id": 549520, "user_id": 1242, "id": 3211843, "testcase_feedback": [], "user": {"name": "abbra", "email": "abokovoy@redhat.com", "id": 1242, "avatar": "https://seccdn.libravatar.org/avatar/3f3c51cb81fb43e685e65ff696d895ac268ff71949128c02b81dec65a4405a0a?s=24&d=retro", "openid": "abbra.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "server-wg"}, {"name": "bind-dyndb-ldap"}, {"name": "gitfreeipa"}, {"name": "gitslapi-nis"}, {"name": "gitfreeipa-docs"}]}}}, {"karma": 0, "comment_id": 3214247, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "> This is a Fedora 38 cloud image with some extra packages installed, so you can install debug symbols, run gdb, etc.\n\nThank you. I got to this point and could reproduce the assert, but the VM with `ipa-getkeytab` does not have a default route. Any idea how to fix that? DHCP assigns `172.27.0.2` for the `eth0` interface, but no default route.", "timestamp": "2023-09-26 09:28:39", "update_id": 549520, "user_id": 1825, "id": 3214247, "testcase_feedback": [], "user": {"name": "fweimer", "email": "fweimer@redhat.com", "id": 1825, "avatar": "https://seccdn.libravatar.org/avatar/24e99cbd8f3721a17417169347326bfdeabd903e02f4733f62cf86e708dad8e2?s=24&d=retro", "openid": "fweimer.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "provenpackager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "centosproject-email-aliases"}, {"name": "gitsecure-coding"}, {"name": "sig-isa"}, {"name": "gitlab-centos-sig-isa"}]}}}, {"karma": 0, "comment_id": 3212782, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "As this needs a client machine which is talking to a configured FreeIPA, and thus at least two VMs which talk to each other, this is unfortunately quite involved. If you have a FreeIPA setup, you could just run the ipa-getkeytab command. From scratch, here is how you can reproduce the cockpit test. Note that this is safe -- they don't run as root, don't create/change permanent files on the host (only temp dirs/files) or change the qemu/libvirt config (all transient VMs with socket networking).\n\nYou can do this in our development toolbox, so that you don't have to install all the nodejs, libvirt and QEMU packages (if you already have them installed, you can skip this):\n```\ntoolbox create --image quay.io/cockpit/tasks -c cockpit\ntoolbox enter cockpit\n```\nThen check out cockpit and build a test image:\n```\ngit clone https://github.com/cockpit-project/cockpit\ncd cockpit\ntest/image-prepare -q fedora-38\n```\nThis is *without* updates-testing still. Now run a FreeIPA test:\n```\ntest/verify/check-system-realms TestKerberos.testNegotiate\n```\nThis ought to succeed. If not, and it's not obvious why (like, missing libvirt packages or so), please ping me here or on Slack, I'm happy to assist.\n\nNow install the glibc update into the VM:\n```\nbots/image-customize -v -r 'dnf upgrade --enablerepo=updates-testing -y --refresh --advisory=FEDORA-2023-7f0a294b1a >&2' fedora-38\n```\n\nNow run the test again, but this time with the `-s` option, which will make it \"sit\" on a test failure without cleaning up the VMs:\n```\ntest/verify/check-system-realms TestKerberos.testNegotiate -s\n```\nThis should fail with this assertion error, and give you some information how to log in via SSH:\n```\nssh -p 2201 -i bots/machine/identity root@127.0.0.2\n```\nMove that terminal to the side -- as soon as you press enter, it'll continue, i.e. clean up all the test VMs.\n\nInside the test VM, you can now reproduce the crash:\n```\n# ipa-getkeytab -p HTTP/x0.cockpit.lan -k /etc/cockpit/krb5.keytab\nKeytab successfully retrieved and stored in: /etc/cockpit/krb5.keytab\nk5_mutex_lock: Received error 22 (Invalid argument)\nipa-getkeytab: ../../include/k5-thread.h:376: k5_mutex_lock: Assertion `r == 0' failed.\nAborted (core dumped)\n```\nThis is a Fedora 38 cloud image with some extra packages installed, so you can install debug symbols, run gdb, etc.", "timestamp": "2023-09-25 05:16:10", "update_id": 549520, "user_id": 3506, "id": 3212782, "testcase_feedback": [], "user": {"name": "martinpitt", "email": "mpitt@redhat.com", "id": 3506, "avatar": "https://seccdn.libravatar.org/avatar/a6a00790fadaca10623f4464437e482d4ec110fe331397506ea9fe2664bec740?s=24&d=retro", "openid": "martinpitt.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "cockpitteam"}, {"name": "ocp-cico-cockpit"}]}}}, {"karma": 1, "comment_id": 3213461, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Works great on my Fedora 38 workstation, I was down until this was applied.", "timestamp": "2023-09-25 18:09:32", "update_id": 549520, "user_id": 7868, "id": 3213461, "testcase_feedback": [], "user": {"name": "jnyuhas", "email": "jnyuhas@gmail.com", "id": 7868, "avatar": "https://seccdn.libravatar.org/avatar/35d7c37f4d664de4694bfbfb0fd3e0dd9d4d15b0f1c4bc9283a67dc053d1fe70?s=24&d=retro", "openid": "jnyuhas.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 0, "comment_id": 3214249, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "Sorry @fweimer, indeed these VMs are offline by default, to make sure none of our tests depends on something outside. Please apply this local hack:\n```diff\n--- test/common/testlib.py\n+++ test/common/testlib.py\n@@ -1290,7 +1290,7 @@ class MachineCase(unittest.TestCase):\n                 if cleanup:\n                     self.addCleanup(network.kill)\n                 self.network = network\n-            networking = self.network.host(restrict=restrict, forward=forward or {})\n+            networking = self.network.host(restrict=False, forward=forward or {})\n             machine = machine_class(verbose=opts.trace, networking=networking, image=image, **kwargs)\n             if opts.fetch and not os.path.exists(machine.image_file):\n                 machine.pull(machine.image_file)\n```", "timestamp": "2023-09-26 09:34:23", "update_id": 549520, "user_id": 3506, "id": 3214249, "testcase_feedback": [], "user": {"name": "martinpitt", "email": "mpitt@redhat.com", "id": 3506, "avatar": "https://seccdn.libravatar.org/avatar/a6a00790fadaca10623f4464437e482d4ec110fe331397506ea9fe2664bec740?s=24&d=retro", "openid": "martinpitt.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "cockpitteam"}, {"name": "ocp-cico-cockpit"}]}}}, {"karma": 0, "comment_id": 3214256, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "But I think I see what's wrong with the current ELF destructor ordering approach. I'll experiment with something else.", "timestamp": "2023-09-26 09:42:33", "update_id": 549520, "user_id": 1825, "id": 3214256, "testcase_feedback": [], "user": {"name": "fweimer", "email": "fweimer@redhat.com", "id": 1825, "avatar": "https://seccdn.libravatar.org/avatar/24e99cbd8f3721a17417169347326bfdeabd903e02f4733f62cf86e708dad8e2?s=24&d=retro", "openid": "fweimer.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "provenpackager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "centosproject-email-aliases"}, {"name": "gitsecure-coding"}, {"name": "sig-isa"}, {"name": "gitlab-centos-sig-isa"}]}}}, {"karma": 0, "comment_id": 3214492, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "With the new approach:\n\n```\n# ipa-getkeytab -p HTTP/x0.cockpit.lan -k /etc/cockpit/krb5.keytab \nKeytab successfully retrieved and stored in: /etc/cockpit/krb5.keytab\n```\n\nI'll do another build, so that the AnyConnect users can test it as well.", "timestamp": "2023-09-26 14:01:59", "update_id": 549520, "user_id": 1825, "id": 3214492, "testcase_feedback": [], "user": {"name": "fweimer", "email": "fweimer@redhat.com", "id": 1825, "avatar": "https://seccdn.libravatar.org/avatar/24e99cbd8f3721a17417169347326bfdeabd903e02f4733f62cf86e708dad8e2?s=24&d=retro", "openid": "fweimer.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "provenpackager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "centosproject-email-aliases"}, {"name": "gitsecure-coding"}, {"name": "sig-isa"}, {"name": "gitlab-centos-sig-isa"}]}}}, {"karma": 0, "comment_id": 3214614, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "I'd love to give you some useful feedback on the new build, but something else has broken cisco's vpn agent in the past week unrelated to this seemingly. I can confirm my system is otherwise functional though.", "timestamp": "2023-09-26 16:31:07", "update_id": 549520, "user_id": 4150, "id": 3214614, "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3215677, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "Many thanks Florian for puzzling this out! Our [nightly run](https://cockpit-logs.us-east-1.linodeobjects.com/pull-0-20230927-012502-172dd31b-fedora-38-updates-testing/log.html) still failed, but dnf only \"saw\" 2.37-6.fc38 on the mirrors still. I'll let you know tomorrow morning. But I'm sure it'll be good, as you tested it on the very thing.", "timestamp": "2023-09-27 05:21:15", "update_id": 549520, "user_id": 3506, "id": 3215677, "testcase_feedback": [], "user": {"name": "martinpitt", "email": "mpitt@redhat.com", "id": 3506, "avatar": "https://seccdn.libravatar.org/avatar/a6a00790fadaca10623f4464437e482d4ec110fe331397506ea9fe2664bec740?s=24&d=retro", "openid": "martinpitt.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "cockpitteam"}, {"name": "ocp-cico-cockpit"}]}}}, {"karma": 0, "comment_id": 3216236, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "Should this work on FC37? I find it doesn't install/change anything on my  system:\n\nNo security updates needed, but 35 updates available\nDependencies resolved.\nNothing to do.\nComplete!\n\nand the problem with the VPN remains", "timestamp": "2023-09-27 17:25:50", "update_id": 549520, "user_id": 7872, "id": 3216236, "testcase_feedback": [], "user": {"name": "chernoff", "email": "chernoff@astro.cornell.edu", "id": 7872, "avatar": "https://seccdn.libravatar.org/avatar/0026fbf9835e8acf61305151e05e467716cf7fb79ab9b1e5d8ff08611f010426?s=24&d=retro", "openid": "chernoff.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": -1, "comment_id": 3216239, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "Applied this to another fc38 system and can confirm the vpn issue remains.", "timestamp": "2023-09-27 17:30:14", "update_id": 549520, "user_id": 4150, "id": 3216239, "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3216246, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "I'm still facing \" Termination reason code 59: Connection attempt failed due to certificate problems\" with cisco anyconnect after upgrade glibc to glibc-2.37-7.fc38.x86_64", "timestamp": "2023-09-27 17:47:27", "update_id": 549520, "user_id": 7873, "id": 3216246, "testcase_feedback": [], "user": {"name": "jostra", "email": "rakovec@gmail.com", "id": 7873, "avatar": "https://seccdn.libravatar.org/avatar/e6611bd0ac1f47ffa11ec869603abf1827b9c77a414d861c7d4579362a24bcb7?s=24&d=retro", "openid": "jostra.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 0, "comment_id": 3221467, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Cockpit's tests are happy again with glibc 2.37-7.fc38, thank you!", "timestamp": "2023-09-28 03:29:48", "update_id": 549520, "user_id": 3506, "id": 3221467, "testcase_feedback": [], "user": {"name": "martinpitt", "email": "mpitt@redhat.com", "id": 3506, "avatar": "https://seccdn.libravatar.org/avatar/a6a00790fadaca10623f4464437e482d4ec110fe331397506ea9fe2664bec740?s=24&d=retro", "openid": "martinpitt.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "cockpitteam"}, {"name": "ocp-cico-cockpit"}]}}}, {"karma": 0, "comment_id": 3222763, "bug_id": 2239304, "comment": {"karma": -1, "karma_critpath": 0, "text": "Thank you for o.m. link \n\"sudo dnf upgrade --enablerepo=updates-testing --refresh --advisory=FEDORA-2023-7f0a294b1a\" \nwhich worked FINE on Sep,19. for glibc 2.37-5.fc38 --> glibc 2.37-6.fc38\n\nsystemd-resolve --status                                        #looked like:\n\nGlobal\n         Protocols: LLMNR=resolve -mDNS -DNSOverTLS DNSSEC=no/unsupported\n  resolv.conf mode: uplink\nCurrent DNS Server: [ IP local-dns1]\n       DNS Servers: [ IP local-dns2] [ IP local-dns3] [ IP local-dns1] #IP local-dns2+3 are given by vpn-connection and Certificate\n        DNS Domain: ~.\n\nAfter update (and to glibc 2.37-7.fc38) it DOESN'T work again!\nand like jostra I'm facing: \"in 2 parts overlayed as before:\nThe certificate on the secure gateway is invalid. A VPN connection will not be established.\nAnyconnect was not able to establish a connection to the specified secure gateway. Please try connecting again.\n--> Cisco AC SMC Ver 4.10.07073\"\n\nsystemd-resolve --status #looks as before\nGlobal\n         Protocols: LLMNR=resolve -mDNS -DNSOverTLS DNSSEC=no/unsupported\n  resolv.conf mode: uplink\n\nLink 2 (enp0s31f6)\n    Current Scopes: none\n         Protocols: -DefaultRoute LLMNR=resolve -mDNS -DNSOverTLS DNSSEC=no/unsupported\n\nLink 3 (wlp0s20f3)\n    Current Scopes: DNS LLMNR/IPv4\n         Protocols: +DefaultRoute LLMNR=resolve -mDNS -DNSOverTLS DNSSEC=no/unsupported\nCurrent DNS Server: [IP local-dns1]\n       DNS Servers: [ IP local-dns1]\n\nLink 4 (virbr0)\n    Current Scopes: none\n         Protocols: -DefaultRoute LLMNR=resolve -mDNS -DNSOverTLS DNSSEC=no/unsupported\n\nand\n\nvpnagentd.service gives errors like:\n\nacvpnagent[108857]: Function: determinePublicAddrCandidateFromDefRoute File: ../../vpn/AgentUtilities/HostConfigMgr.cpp Line: 3057 Invoked Function: CHos>\nacvpnagent[108857]: Function: updatePotentialPublicAddresses File: ../../vpn/AgentUtilities/HostConfigMgr.cpp Line: 3190 Invoked Function: CHostConfigMgr>\nacvpnagent[108857]: Function: GetSettings File: ../../vpn/Agent/ServicePluginMgr.cpp Line: 289 m_pIServicePlugin is NULL\nacvpnagent[108857]: Function: GetSettings File: ../../vpn/Agent/ServicePluginMgr.cpp Line: 289 m_pIServicePlugin is NULL\nacvpnagent[108857]: Function: GetSettings File: ../../vpn/Agent/ServicePluginMgr.cpp Line: 289 m_pIServicePlugin is NULL\nacvpnagent[108857]: Function: GetSettings File: ../../vpn/Agent/ServicePluginMgr.cpp Line: 289 m_pIServicePlugin is NULL\nacvpnagent[108857]: Function: GetSettings File: ../../vpn/Agent/ServicePluginMgr.cpp Line: 289 m_pIServicePlugin is NULL\nacvpnagent[108857]: Function: OnIpcMessageReceived File: ../../vpn/Common/IPC/IPCDepot.cpp Line: 1240 Invoked Function: CIpcTransport::OnSocketReadComplete Return Code: -33292279 (0xFE040009) Description: IPCTRANSPORT_ERROR_UNEXPECTED remote peer: gui\n\nI kindly ask you to provide a final solution!\n\nThank you in advance!", "timestamp": "2023-09-28 05:50:57", "update_id": 549520, "user_id": 7875, "id": 3222763, "testcase_feedback": [], "user": {"name": "mpfeiler", "email": "m_pfeiler@gmx.at", "id": 7875, "avatar": "https://seccdn.libravatar.org/avatar/6d286952d5448038e23d14ad75f5989af1d040b507e939202daa13b26b38ac05?s=24&d=retro", "openid": "mpfeiler.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 0, "comment_id": 3222948, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "thanks for working out the crash, @fweimer , sorry I didn't respond - I'm bad at checking bodhi email. in future feel free to ping me on matrix. I can fiddle with openqa to disable the limits and get a backtrace in this kinda situation if you need it.", "timestamp": "2023-09-28 06:10:34", "update_id": 549520, "user_id": 302, "id": 3222948, "testcase_feedback": [], "user": {"name": "adamwill", "email": "awilliam@redhat.com", "id": 302, "avatar": "https://seccdn.libravatar.org/avatar/51720fbb2ec5dfd2bc005ac374a3bfd3190cb8f45f3bb8d6bdf35fb6a051d03b?s=24&d=retro", "openid": "adamwill.id.fedoraproject.org", "groups": [{"name": "proventesters"}, {"name": "server-wg"}, {"name": "provenpackager"}, {"name": "packager"}, {"name": "qa-tools-sig"}, {"name": "fedora-contributor"}, {"name": "news"}, {"name": "gittriage"}, {"name": "sysadmin"}, {"name": "gitspin-kickstarts"}, {"name": "signed_fpca"}, {"name": "gitgeneric-release"}, {"name": "yak_farmers"}, {"name": "triagers"}, {"name": "qa"}, {"name": "sysadmin-qa"}, {"name": "fedorabugs"}, {"name": "ipausers"}, {"name": "gitmkinitrd"}, {"name": "qa-admin"}, {"name": "aws-qa"}, {"name": "change-wranglers"}, {"name": "fedora-ci-admins"}, {"name": "common-issues-triage"}, {"name": "sysadmin-main"}, {"name": "program-management"}, {"name": "gitfedora-project-schedule"}]}}}, {"karma": 0, "comment_id": 3205781, "bug_id": 2239304, "comment": {"karma": -1, "karma_critpath": 0, "text": "The openQA failure here seems to be reproducible - it has failed the same way four times (twice each on prod and stg). The test that's failing is deployment of a FreeIPA replica (deployment of the first server instance works fine). The logs show:\n\n    2023-09-19T16:11:40Z DEBUG args=['/usr/sbin/ipa-getkeytab', '-k', '/etc/dirsrv/ds.keytab', '-p', 'ldap/ipa003.test.openqa.fedoraproject.org@TEST.OPENQA.FEDORAPROJECT.ORG', '-H', 'ldaps://ipa\n    002.test.openqa.fedoraproject.org']\n    2023-09-19T16:11:41Z DEBUG Process finished, return code=-6\n    2023-09-19T16:11:41Z DEBUG stdout=\n    2023-09-19T16:11:41Z DEBUG stderr=Keytab successfully retrieved and stored in: /etc/dirsrv/ds.keytab\n    k5_mutex_lock: Received error 22 (Invalid argument)\n    ipa-getkeytab: ../../include/k5-thread.h:376: k5_mutex_lock: Assertion `r == 0' failed.", "timestamp": "2023-09-19 19:26:39", "update_id": 549520, "user_id": 302, "id": 3205781, "testcase_feedback": [], "user": {"name": "adamwill", "email": "awilliam@redhat.com", "id": 302, "avatar": "https://seccdn.libravatar.org/avatar/51720fbb2ec5dfd2bc005ac374a3bfd3190cb8f45f3bb8d6bdf35fb6a051d03b?s=24&d=retro", "openid": "adamwill.id.fedoraproject.org", "groups": [{"name": "proventesters"}, {"name": "server-wg"}, {"name": "provenpackager"}, {"name": "packager"}, {"name": "qa-tools-sig"}, {"name": "fedora-contributor"}, {"name": "news"}, {"name": "gittriage"}, {"name": "sysadmin"}, {"name": "gitspin-kickstarts"}, {"name": "signed_fpca"}, {"name": "gitgeneric-release"}, {"name": "yak_farmers"}, {"name": "triagers"}, {"name": "qa"}, {"name": "sysadmin-qa"}, {"name": "fedorabugs"}, {"name": "ipausers"}, {"name": "gitmkinitrd"}, {"name": "qa-admin"}, {"name": "aws-qa"}, {"name": "change-wranglers"}, {"name": "fedora-ci-admins"}, {"name": "common-issues-triage"}, {"name": "sysadmin-main"}, {"name": "program-management"}, {"name": "gitfedora-project-schedule"}]}}}, {"karma": 0, "comment_id": 3205782, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "Looks like we didn't get a coredump because of resource limits, from the system journal :(", "timestamp": "2023-09-19 19:28:44", "update_id": 549520, "user_id": 302, "id": 3205782, "testcase_feedback": [], "user": {"name": "adamwill", "email": "awilliam@redhat.com", "id": 302, "avatar": "https://seccdn.libravatar.org/avatar/51720fbb2ec5dfd2bc005ac374a3bfd3190cb8f45f3bb8d6bdf35fb6a051d03b?s=24&d=retro", "openid": "adamwill.id.fedoraproject.org", "groups": [{"name": "proventesters"}, {"name": "server-wg"}, {"name": "provenpackager"}, {"name": "packager"}, {"name": "qa-tools-sig"}, {"name": "fedora-contributor"}, {"name": "news"}, {"name": "gittriage"}, {"name": "sysadmin"}, {"name": "gitspin-kickstarts"}, {"name": "signed_fpca"}, {"name": "gitgeneric-release"}, {"name": "yak_farmers"}, {"name": "triagers"}, {"name": "qa"}, {"name": "sysadmin-qa"}, {"name": "fedorabugs"}, {"name": "ipausers"}, {"name": "gitmkinitrd"}, {"name": "qa-admin"}, {"name": "aws-qa"}, {"name": "change-wranglers"}, {"name": "fedora-ci-admins"}, {"name": "common-issues-triage"}, {"name": "sysadmin-main"}, {"name": "program-management"}, {"name": "gitfedora-project-schedule"}]}}}, {"karma": 0, "comment_id": 3206912, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "Judging by the errors, it is the code in https://github.com/krb5/krb5/blob/krb5-1.21.1-final/src/lib/krb5/keytab/kt_file.c#L523-L552 which is a file-based keytab backend. The locking mutex is local to this code. `ipa-getkeytab` is a single-process program that uses `krb5_kt_add_entry()` function to store an entry into a keytab.", "timestamp": "2023-09-20 08:23:46", "update_id": 549520, "user_id": 1242, "id": 3206912, "testcase_feedback": [], "user": {"name": "abbra", "email": "abokovoy@redhat.com", "id": 1242, "avatar": "https://seccdn.libravatar.org/avatar/3f3c51cb81fb43e685e65ff696d895ac268ff71949128c02b81dec65a4405a0a?s=24&d=retro", "openid": "abbra.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "server-wg"}, {"name": "bind-dyndb-ldap"}, {"name": "gitfreeipa"}, {"name": "gitslapi-nis"}, {"name": "gitfreeipa-docs"}]}}}, {"karma": 1, "comment_id": 3208247, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "It works for me as well!\nThank you!", "timestamp": "2023-09-20 19:59:04", "update_id": 549520, "user_id": 7863, "id": 3208247, "testcase_feedback": [], "user": {"name": "reanimator", "email": "aleksandrs.gumenuks@live.com", "id": 7863, "avatar": "https://seccdn.libravatar.org/avatar/b4956284229c3fb1f6eab51a9b34c2977d15aa77027c99d034bbc4f936978576?s=24&d=retro", "openid": "reanimator.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 1, "comment_id": 3208481, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Can confirm it resolves the issue  with cisco secure client/anyconnect.", "timestamp": "2023-09-21 01:03:08", "update_id": 549520, "user_id": 4150, "id": 3208481, "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3208859, "bug_id": 2239304, "comment": {"karma": -1, "karma_critpath": 0, "text": "Crashes confirmed with [cockpit tests](https://cockpit-logs.us-east-1.linodeobjects.com/pull-0-20230921-012401-ce07ea73-fedora-38-updates-testing/log.html), they now segfault `ipa-getkeytab` and `iscsid` at least. \n\nThe [journal for ipa-getkeytab crash](https://cockpit-logs.us-east-1.linodeobjects.com/pull-0-20230921-012401-ce07ea73-fedora-38-updates-testing/TestIPA-testClientCertAuthentication-fedora-38-127.0.0.2-2501-FAIL.log.gz) shows the crash in `krb5int_key_delete()`, and the [test output](https://cockpit-logs.us-east-1.linodeobjects.com/pull-0-20230921-012401-ce07ea73-fedora-38-updates-testing/log.html#288-2) confirms the assertion:\n```\nipa-getkeytab: ../../include/k5-thread.h:376: k5_mutex_lock: Assertion `r == 0' failed.\n```\n\nThe [journal for the iscsi crash](https://cockpit-logs.us-east-1.linodeobjects.com/pull-0-20230921-012401-ce07ea73-fedora-38-updates-testing/TestStorageISCSI-testISCSI-fedora-38-127.0.0.2-2301-FAIL.log.gz) doesn't even get that far, it crashes right at the beginning:\n```\nsystemd-coredump[34279]: Process 34270 (iscsid) of user 0 dumped core.    \n                             \nModule libpcre2-8.so.0 from rpm pcre2-10.42-1.fc38.1.x86_64    \nModule liblz4.so.1 from rpm lz4-1.9.4-2.fc38.x86_64    \nModule libcap.so.2 from rpm libcap-2.48-6.fc38.x86_64    \nModule liblzma.so.5 from rpm xz-5.4.1-1.fc38.x86_64    \nModule libzstd.so.1 from rpm zstd-1.5.5-1.fc38.x86_64    \nModule libselinux.so.1 from rpm libselinux-3.5-1.fc38.x86_64    \nModule libblkid.so.1 from rpm util-linux-2.38.1-4.fc38.x86_64    \nModule libz.so.1 from rpm zlib-1.2.13-3.fc38.x86_64    \nModule libopeniscsiusr.so.0.2.0 from rpm iscsi-initiator-utils-6.2.1.4-10.git2a8f9d8.fc38.x86_64    \nModule libsystemd.so.0 from rpm systemd-253.10-1.fc38.x86_64    \nModule libkmod.so.2 from rpm kmod-30-4.fc38.x86_64    \nModule libmount.so.1 from rpm util-linux-2.38.1-4.fc38.x86_64    \nModule libcrypto.so.3 from rpm openssl-3.0.9-2.fc38.x86_64    \nModule libisns.so.0 from rpm isns-utils-0.101-6.fc38.x86_64                             \nModule iscsid from rpm iscsi-initiator-utils-6.2.1.4-10.git2a8f9d8.fc38.x86_64                             \nStack trace of thread 34270:                             \n#0  0x00007fc7ef527324 __poll (libc.so.6 + 0x105324)                             \n#1  0x0000558cec9fcbc5 main (iscsid + 0x8bc5)                             \n#2  0x00007fc7ef449b8a __libc_start_call_main (libc.so.6 + 0x27b8a)                             \n#3  0x00007fc7ef449c4b __libc_start_main@@GLIBC_2.34 (libc.so.6 + 0x27c4b)                             \n#4  0x0000558cec9fd805 _start (iscsid + 0x9805)                             \nELF object binary architecture: AMD x86-64                             \nsystemd[1]: iscsid.service: Main process exited, code=dumped, status=6/ABRT                    \n```", "timestamp": "2023-09-21 04:36:30", "update_id": 549520, "user_id": 3506, "id": 3208859, "testcase_feedback": [], "user": {"name": "martinpitt", "email": "mpitt@redhat.com", "id": 3506, "avatar": "https://seccdn.libravatar.org/avatar/a6a00790fadaca10623f4464437e482d4ec110fe331397506ea9fe2664bec740?s=24&d=retro", "openid": "martinpitt.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}, {"name": "cockpitteam"}, {"name": "ocp-cico-cockpit"}]}}}, {"karma": 0, "comment_id": 3254059, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Works for me too, thanks very much.", "timestamp": "2023-10-21 20:57:47", "update_id": 558726, "user_id": 7920, "id": 3254059, "testcase_feedback": [], "user": {"name": "daithiociagain", "email": "dksw.daithi@gmail.com", "id": 7920, "avatar": "https://seccdn.libravatar.org/avatar/7f1bedc2ba234e66f13d026687118bdc66e7af1cce559f480686ea3cc2b34586?s=24&d=retro", "openid": "daithiociagain.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 1, "comment_id": 3253300, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Can confirm cisco secureclient works as expected.", "timestamp": "2023-10-20 23:01:23", "update_id": 558726, "user_id": 4150, "id": 3253300, "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3232553, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "", "timestamp": "2023-10-03 20:24:57", "update_id": 554728, "user_id": 3703, "id": 3232553, "testcase_feedback": [], "user": {"name": "markec", "email": "marko.bevc@gmail.com", "id": 3703, "avatar": "https://seccdn.libravatar.org/avatar/2a3640b3b2df97a062aff29758dd6a2ce98adba2cdd4da3fceb85df19b25200c?s=24&d=retro", "openid": "markec.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": -1, "comment_id": 3232611, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Issue with cisco vpn still present, seems to work otherwise.", "timestamp": "2023-10-03 21:16:46", "update_id": 554728, "user_id": 4150, "id": 3232611, "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3233519, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "", "timestamp": "2023-10-04 08:08:30", "update_id": 554728, "user_id": 3307, "id": 3233519, "testcase_feedback": [], "user": {"name": "paulblack", "email": "paul.0000.black@gmail.com", "id": 3307, "avatar": "https://seccdn.libravatar.org/avatar/92d7af1ece6d2ab8a0e94d1e7b63d991b68d34106fb98d33c4076c339a2cf2b7?s=24&d=retro", "openid": "paulblack.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 0, "comment_id": 3233708, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "I just waived the automated test failures, but I don't see my waiver comment here, so:\n\n1. The scratch build validation failure is an strace build failure that also occurs on baseline; waived\n2. The debugging symbols in libc.so.6 and ld.so is expected; waived\n3. The cope build validation failure seems to be an infrastructure issue; waived", "timestamp": "2023-10-04 11:26:37", "update_id": 554728, "user_id": 3654, "id": 3233708, "testcase_feedback": [], "user": {"name": "submachine", "email": "arjun.is@lostca.se", "id": 3654, "avatar": "https://seccdn.libravatar.org/avatar/0024df7e94ab8e2363ae4c9371bcf3d8a0bfa02bb659cb7a0f1ec9e2e592c8b4?s=24&d=retro", "openid": "submachine.id.fedoraproject.org", "groups": [{"name": "packager"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}]}}}, {"karma": 0, "comment_id": 3233721, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "I can confirm that the cisco secure connect (anyconnect) vpn problem is still present, otherwise everything seems to work.", "timestamp": "2023-10-04 11:31:56", "update_id": 554728, "user_id": 7886, "id": 3233721, "testcase_feedback": [], "user": {"name": "didier", "email": "dwiroth@gmail.com", "id": 7886, "avatar": "https://seccdn.libravatar.org/avatar/a9425531a034fea15318f8d1d84c6c7f75b4d644b29da79a1bedab972e01a4e8?s=24&d=retro", "openid": "didier.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 0, "comment_id": 3254810, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Also worked for me on F38. I did not test F37 or F39.", "timestamp": "2023-10-23 02:45:42", "update_id": 558726, "user_id": 7924, "id": 3254810, "testcase_feedback": [], "user": {"name": "jimstaffer", "email": "jim@yosj.com.au", "id": 7924, "avatar": "https://seccdn.libravatar.org/avatar/c77b57c0c59746534b7e151efee07e6327325e3e83d76a900ef6507ac63fe5ca?s=24&d=retro", "openid": "jimstaffer.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 1, "comment_id": 3239444, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "", "timestamp": "2023-10-09 03:36:37", "update_id": 555559, "user_id": 7885, "id": 3239444, "testcase_feedback": [], "user": {"name": "jnk0", "email": "fedora.mx@jules.de", "id": 7885, "avatar": "https://seccdn.libravatar.org/avatar/5cd480c5639b391c2a8aeac357534526c58b378d227229ca9a956e6e166b0a91?s=24&d=retro", "openid": "jnk0.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 1, "comment_id": 3237672, "bug_id": 2239304, "comment": {"karma": 1, "karma_critpath": 0, "text": "Works and fixes the bug described in BZ#2239304. Cisco AnyConnect can be used in F39.", "timestamp": "2023-10-07 11:31:38", "update_id": 555559, "user_id": 7885, "id": 3237672, "testcase_feedback": [], "user": {"name": "jnk0", "email": "fedora.mx@jules.de", "id": 7885, "avatar": "https://seccdn.libravatar.org/avatar/5cd480c5639b391c2a8aeac357534526c58b378d227229ca9a956e6e166b0a91?s=24&d=retro", "openid": "jnk0.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3239566, "bug_id": 2239304, "comment": {"karma": 0, "karma_critpath": 0, "text": "cisco anyconnect with glibc-2.37-10.fc38 still doesn't work!!", "timestamp": "2023-10-09 08:25:26", "update_id": 554728, "user_id": 7873, "id": 3239566, "testcase_feedback": [], "user": {"name": "jostra", "email": "rakovec@gmail.com", "id": 7873, "avatar": "https://seccdn.libravatar.org/avatar/e6611bd0ac1f47ffa11ec869603abf1827b9c77a414d861c7d4579362a24bcb7?s=24&d=retro", "openid": "jostra.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}]}, {"bug_id": 2244992, "title": "glibc: Improve compatibility of ELF destructor ordering", "security": false, "parent": false, "feedback": [{"karma": 0, "comment_id": 3289601, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "No problems so far.", "timestamp": "2023-11-22 05:38:47", "update_id": 565410, "user_id": 4756, "id": 3289601, "testcase_feedback": [], "user": {"name": "nixuser", "email": "nixuser@mail.com", "id": 4756, "avatar": "https://seccdn.libravatar.org/avatar/365861a25716b355bf17bd01d55d25c29e1d98a79e197ef46bd8cfafdd93c836?s=24&d=retro", "openid": "nixuser.id.fedoraproject.org", "groups": [{"name": "qa"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}]}}}, {"karma": 0, "comment_id": 3254059, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "Works for me too, thanks very much.", "timestamp": "2023-10-21 20:57:47", "update_id": 558726, "user_id": 7920, "id": 3254059, "testcase_feedback": [], "user": {"name": "daithiociagain", "email": "dksw.daithi@gmail.com", "id": 7920, "avatar": "https://seccdn.libravatar.org/avatar/7f1bedc2ba234e66f13d026687118bdc66e7af1cce559f480686ea3cc2b34586?s=24&d=retro", "openid": "daithiociagain.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 1, "comment_id": 3253300, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "Can confirm cisco secureclient works as expected.", "timestamp": "2023-10-20 23:01:23", "update_id": 558726, "user_id": 4150, "id": 3253300, "testcase_feedback": [], "user": {"name": "snehring", "email": "snehring@iastate.edu", "id": 4150, "avatar": "https://seccdn.libravatar.org/avatar/22c865669b603fe58f5cf4c04b49d1e59252fb20e7ae89f8ff03b686f6d72650?s=24&d=retro", "openid": "snehring.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3254810, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "Also worked for me on F38. I did not test F37 or F39.", "timestamp": "2023-10-23 02:45:42", "update_id": 558726, "user_id": 7924, "id": 3254810, "testcase_feedback": [], "user": {"name": "jimstaffer", "email": "jim@yosj.com.au", "id": 7924, "avatar": "https://seccdn.libravatar.org/avatar/c77b57c0c59746534b7e151efee07e6327325e3e83d76a900ef6507ac63fe5ca?s=24&d=retro", "openid": "jimstaffer.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}, {"karma": 1, "comment_id": 3254887, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "Works well and fixes the problems with Cisco Secure Client.", "timestamp": "2023-10-23 04:18:19", "update_id": 558721, "user_id": 7885, "id": 3254887, "testcase_feedback": [], "user": {"name": "jnk0", "email": "fedora.mx@jules.de", "id": 7885, "avatar": "https://seccdn.libravatar.org/avatar/5cd480c5639b391c2a8aeac357534526c58b378d227229ca9a956e6e166b0a91?s=24&d=retro", "openid": "jnk0.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "signed_fpca"}]}}}, {"karma": 0, "comment_id": 3287407, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "", "timestamp": "2023-11-20 14:33:59", "update_id": 565410, "user_id": 2935, "id": 3287407, "testcase_feedback": [], "user": {"name": "imabug", "email": "eugenemah@gmail.com", "id": 2935, "avatar": "https://seccdn.libravatar.org/avatar/ea8ba927783ee0ef10e6f05f14791abf925972bac4f31aba458058d3e7907a6d?s=24&d=retro", "openid": "imabug.id.fedoraproject.org", "groups": [{"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedora-hams"}]}}}, {"karma": 0, "comment_id": 3289579, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "No problems so far.", "timestamp": "2023-11-22 04:35:24", "update_id": 565422, "user_id": 4756, "id": 3289579, "testcase_feedback": [], "user": {"name": "nixuser", "email": "nixuser@mail.com", "id": 4756, "avatar": "https://seccdn.libravatar.org/avatar/365861a25716b355bf17bd01d55d25c29e1d98a79e197ef46bd8cfafdd93c836?s=24&d=retro", "openid": "nixuser.id.fedoraproject.org", "groups": [{"name": "qa"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}]}}}, {"karma": 0, "comment_id": 3289603, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "Working fine here.", "timestamp": "2023-11-22 05:41:13", "update_id": 565409, "user_id": 4756, "id": 3289603, "testcase_feedback": [], "user": {"name": "nixuser", "email": "nixuser@mail.com", "id": 4756, "avatar": "https://seccdn.libravatar.org/avatar/365861a25716b355bf17bd01d55d25c29e1d98a79e197ef46bd8cfafdd93c836?s=24&d=retro", "openid": "nixuser.id.fedoraproject.org", "groups": [{"name": "qa"}, {"name": "ipausers"}, {"name": "fedora-contributor"}, {"name": "signed_fpca"}, {"name": "fedorabugs"}]}}}, {"karma": 0, "comment_id": 3293458, "bug_id": 2244992, "comment": {"karma": 1, "karma_critpath": 0, "text": "", "timestamp": "2023-11-26 15:42:30", "update_id": 565410, "user_id": 2998, "id": 3293458, "testcase_feedback": [], "user": {"name": "samoht0", "email": "samoht0-bugzilla@yahoo.com", "id": 2998, "avatar": "https://seccdn.libravatar.org/avatar/4a429e081c6c9f7b9d9cd4a3f4037fc93517c13e17258f8549f2d2f85f0912e7?s=24&d=retro", "openid": "samoht0.id.fedoraproject.org", "groups": [{"name": "ipausers"}]}}}]}], "updateid": "FEDORA-2023-7dfa1f6429", "karma": 5, "content_type": "rpm", "test_cases": []}, "can_edit": false, "ci_allowed": false}