From 32550560a1b6e00565db8d995c84f304d4cb9893 Mon Sep 17 00:00:00 2001 From: Tim Rozet Date: Mon, 5 Mar 2018 17:08:03 -0500 Subject: [PATCH] Fixes ceph key import failures There is an issue with HA deployments where sometimes key imports fail for Ceph which seem to occur around 50% of the time. When logging in after a failure, the key import seems to work which indicates it may be a race condition. In addition, sometimes the keyring that is created is missing the "caps" section of the file, which will also fail import. This patch adds a retries for a minute to try to import the key. It also moves creating/importing to the same Exec because there is evidence that the file is being modified by some other process right after the file content is created in the previous exec. JIRA: APEX-563 Change-Id: Ie8cfeb4803f6bed95f9e612eeb37c5cdf2d76617 Signed-off-by: Tim Rozet --- build/overcloud-full.sh | 2 ++ build/patches/puppet-ceph.patch | 76 +++++++++++++++++++++++++++++++++++++++++ 2 files changed, 78 insertions(+) create mode 100644 build/patches/puppet-ceph.patch diff --git a/build/overcloud-full.sh b/build/overcloud-full.sh index 527e39ea..e50fc863 100755 --- a/build/overcloud-full.sh +++ b/build/overcloud-full.sh @@ -145,6 +145,8 @@ LIBGUESTFS_BACKEND=direct $VIRT_CUSTOMIZE \ --install python-etcd,puppet-etcd \ --install patch \ --install docker,kubelet,kubeadm,kubectl,kubernetes-cni \ + --upload ${BUILD_ROOT}/patches/puppet-ceph.patch:/etc/puppet/modules/ceph/ \ + --run-command "cd /etc/puppet/modules/ceph && patch -p1 < puppet-ceph.patch" \ -a overcloud-full_build.qcow2 # upload and install barometer packages diff --git a/build/patches/puppet-ceph.patch b/build/patches/puppet-ceph.patch new file mode 100644 index 00000000..18bf9ee4 --- /dev/null +++ b/build/patches/puppet-ceph.patch @@ -0,0 +1,76 @@ +From 99a0bcc818ed801f6cb9e07a9904ee40e624bdab Mon Sep 17 00:00:00 2001 +From: Tim Rozet +Date: Mon, 5 Mar 2018 17:03:00 -0500 +Subject: [PATCH] Fixes ceph key import failures by adding multiple attempts + +Signed-off-by: Tim Rozet +--- + manifests/key.pp | 42 +++++++++++++++++------------------------- + 1 file changed, 17 insertions(+), 25 deletions(-) + +diff --git a/manifests/key.pp b/manifests/key.pp +index 911df1a..d47a4c3 100644 +--- a/manifests/key.pp ++++ b/manifests/key.pp +@@ -123,22 +123,6 @@ define ceph::key ( + } + } + +- # ceph-authtool --add-key is idempotent, will just update pre-existing keys +- exec { "ceph-key-${name}": +- command => "/bin/true # comment to satisfy puppet syntax requirements +-set -ex +-ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps}", +- unless => "/bin/true # comment to satisfy puppet syntax requirements +-set -x +-NEW_KEYRING=\$(mktemp) +-ceph-authtool \$NEW_KEYRING --name '${name}' --add-key '${secret}' ${caps} +-diff -N \$NEW_KEYRING ${keyring_path} +-rv=\$? +-rm \$NEW_KEYRING +-exit \$rv", +- require => [ File[$keyring_path], ], +- logoutput => true, +- } + + if $inject { + +@@ -162,18 +146,26 @@ exit \$rv", + exec { "ceph-injectkey-${name}": + command => "/bin/true # comment to satisfy puppet syntax requirements + set -ex ++cat ${keyring_path} ++ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps} ++cat ${keyring_path} + ceph ${cluster_option} ${inject_id_option} ${inject_keyring_option} auth import -i ${keyring_path}", +- unless => "/bin/true # comment to satisfy puppet syntax requirements +-set -x +-OLD_KEYRING=\$(mktemp) +-ceph ${cluster_option} ${inject_id_option} ${inject_keyring_option} auth get ${name} -o \$OLD_KEYRING || true +-diff -N \$OLD_KEYRING ${keyring_path} +-rv=$? +-rm \$OLD_KEYRING +-exit \$rv", +- require => [ Class['ceph'], Exec["ceph-key-${name}"], ], ++ require => [ File[$keyring_path], Class['ceph'] ], + logoutput => true, ++ tries => 6, ++ try_sleep => 10 + } + ++ } else { ++ ++ # ceph-authtool --add-key is idempotent, will just update pre-existing keys ++ exec { "ceph-key-${name}": ++ command => "/bin/true # comment to satisfy puppet syntax requirements ++set -ex ++ceph-authtool ${keyring_path} --name '${name}' --add-key '${secret}' ${caps} ++cat ${keyring_path}", ++ require => [ File[$keyring_path], ], ++ logoutput => true, ++ } + } + } +-- +2.14.3 + -- 2.16.6