xci: Fix checkout location for OpenStack-Ansible

The OPENSTACK_OSA_PATH only makes sense on localhost. As such, when we
use it on playbooks that operate on remote hosts, the result is not
predictable. However, we rsync the entire releng-xci repository to the
opfnv host so we can make everything predictable by simply clone
everything in advance in the .cache directory. That directory is then
rsync'd to the opnfv host. As such, we can repurpose the
OPENSTACK_OSA_PATH to point to the path into the OPNFV host. Moreover,
all external repositories are being cloned to .cache/repos so we can
eliminate some variables in order to simplify the code. Finally, we
bring back the ability to use an external OSA repository for
development purposes.

Change-Id: Ieef3e22ae2085f6735185634d555cfc0d4b69b39
Signed-off-by: Markos Chandras <mchandras@suse.de>

diff --git a/xci/config/env-vars b/xci/config/env-vars
index 67ce443..3777bec 100755 (executable)
--- a/xci/config/env-vars
+++ b/xci/config/env-vars
@@ -11,15 +11,17 @@ export OPNFV_HOST_IP=192.168.122.2
 export XCI_FLAVOR_ANSIBLE_FILE_PATH=$XCI_PATH/xci/file/$XCI_FLAVOR
 export CI_LOOP=${CI_LOOP:-daily}
 export JOB_NAME=${JOB_NAME:-false}
+# XCI_CACHE is a cache on localhost where repositories and scenarios are cloned.
 export XCI_CACHE=${XCI_PATH}/.cache
+# OPNFV_XCI_CACHE is similar to XCI_CACHE but refers to the remote OPNFV host.
+export OPNFV_XCI_CACHE="/root/releng-xci/.cache"
 export XCI_SCENARIOS_CACHE="${XCI_CACHE}/repos/scenarios"
 export XCI_PLAYBOOKS=${XCI_PATH}/xci/playbooks
 
 #-------------------------------------------------------------------------------
-# Paths where git repositories of XCI Components will be cloned
+# Paths where git repositories of XCI Components will be cloned on the OPNFV host
 #-------------------------------------------------------------------------------
-export OPENSTACK_BIFROST_PATH="${XCI_CACHE}/repos/bifrost"
-export OPENSTACK_OSA_PATH="${XCI_CACHE}/repos/openstack-ansible"
+export OPENSTACK_OSA_PATH="${OPNFV_XCI_CACHE}/repos/openstack-ansible"
 
 # Logging
 export LOG_PATH=${LOG_PATH:-${XCI_PATH}/xci/logs}

diff --git a/xci/playbooks/configure-localhost.yml b/xci/playbooks/configure-localhost.yml
index 0265fd2..279ae16 100644 (file)
--- a/xci/playbooks/configure-localhost.yml
+++ b/xci/playbooks/configure-localhost.yml
@@ -12,14 +12,7 @@
   vars_files:
     - ../var/opnfv.yml
 
-  roles:
-    - role: clone-repository
-      project: "openstack/openstack-ansible-openstack_openrc"
-      repo: "{{ OPENSTACK_OSA_OPENRC_GIT_URL }}"
-      dest: roles/openstack-ansible-openstack_openrc
-      version: "master"
-
-  tasks:
+  pre_tasks:
     - name: Load distribution variables
       include_vars:
         file: ../var/{{ ansible_os_family }}.yml
@@ -29,11 +22,23 @@
         state: absent
         recurse: no
       with_items:
-        - "{{ OPENSTACK_BIFROST_PATH }}"
-        - "{{ OPENSTACK_OSA_PATH }}"
-        - "{{ OPENSTACK_OSA_ETC_PATH }}"
+        - "{{ XCI_CACHE }}/repos"
         - "{{ LOG_PATH }} "
         - "{{ OPNFV_SSH_HOST_KEYS_PATH }}"
+
+  roles:
+    - role: clone-repository
+      project: "openstack/openstack-ansible-openstack_openrc"
+      repo: "{{ OPENSTACK_OSA_OPENRC_GIT_URL }}"
+      dest: roles/openstack-ansible-openstack_openrc
+      version: "master"
+    - role: clone-repository
+      project: "openstack/openstack-ansible"
+      repo: "{{ OPENSTACK_OSA_GIT_URL }}"
+      dest: "{{ XCI_CACHE }}/repos/openstack-ansible"
+      version: "{{ OPENSTACK_OSA_VERSION }}"
+
+  tasks:
     - name:  create log directory {{LOG_PATH}}
       file:
         path: "{{LOG_PATH}}"
@@ -56,3 +61,12 @@
     - name: generate self signed certificate
       command: openssl req -new -nodes -x509 -subj "{{ XCI_SSL_SUBJECT }}" -days 3650 -keyout "/etc/ssl/private/xci.key" -out "/etc/ssl/certs/xci.crt" -extensions v3_ca
       become: true
+    - name: Synchronize local development OSA repository to XCI paths
+      # command module is much faster than the copy module
+      synchronize:
+        src: "{{ OPENSTACK_OSA_DEV_PATH }}"
+        dest: "{{ XCI_CACHE }}/repos/openstack-ansible"
+        recursive: yes
+        delete: yes
+      when:
+        - OPENSTACK_OSA_DEV_PATH != ""

diff --git a/xci/playbooks/configure-opnfvhost.yml b/xci/playbooks/configure-opnfvhost.yml
index e9e4f6b..0ceacc0 100644 (file)
--- a/xci/playbooks/configure-opnfvhost.yml
+++ b/xci/playbooks/configure-opnfvhost.yml
@@ -7,23 +7,6 @@
 # which accompanies this distribution, and is available at
 # http://www.apache.org/licenses/LICENSE-2.0
 ##############################################################################
-- hosts: opnfv
-  remote_user: root
-  vars_files:
-    - ../var/opnfv.yml
-  pre_tasks:
-    - name: Load distribution variables
-      include_vars:
-        file: ../var/{{ ansible_os_family }}.yml
-        file: ../file/{{ XCI_FLAVOR }}/flavor-vars.yml
-
-  roles:
-    - role: clone-repository
-      project: "openstack/openstack-ansible"
-      repo: "{{ OPENSTACK_OSA_GIT_URL }}"
-      dest: "{{ OPENSTACK_OSA_PATH }}"
-      version: "{{ OPENSTACK_OSA_VERSION }}"
-
 - hosts: opnfv
   remote_user: root
   vars_files:
@@ -35,9 +18,14 @@
       with_items:
         - ../var/{{ ansible_os_family }}.yml
         - ../file/{{ XCI_FLAVOR }}/flavor-vars.yml
+    - name: Set facts for remote deployment
+      set_fact:
+        remote_xci_path: "{{ ansible_env.HOME }}/releng-xci"
+        remote_xci_flavor_files: "{{ ansible_env.HOME }}/releng-xci/xci/file/{{ XCI_FLAVOR }}"
+        remote_xci_playbooks: "{{ ansible_env.HOME }}/releng-xci/xci/playbooks"
 
   roles:
-      - role: configure-network
+    - role: configure-network
 
   tasks:
     - name: generate SSH keys
@@ -49,11 +37,6 @@
         src: "{{ ansible_env.HOME }}/.ssh/id_rsa.pub"
         dest: "../file/authorized_keys"
         flat: yes
-    - name: Set facts for remote deployment
-      set_fact:
-        remote_xci_path: "{{ ansible_env.HOME }}/releng-xci"
-        remote_xci_flavor_files: "{{ ansible_env.HOME }}/releng-xci/xci/file/{{ XCI_FLAVOR }}"
-        remote_xci_playbooks: "{{ ansible_env.HOME }}/releng-xci/xci/playbooks"
     - name: Copy releng-xci to remote host
       synchronize:
         src: "{{ XCI_PATH }}/"

diff --git a/xci/playbooks/provision-vm-nodes.yml b/xci/playbooks/provision-vm-nodes.yml
index 8e91741..8b8bb30 100644 (file)
--- a/xci/playbooks/provision-vm-nodes.yml
+++ b/xci/playbooks/provision-vm-nodes.yml
@@ -17,7 +17,11 @@
       include_vars:
         file: ../var/{{ ansible_os_family }}.yml
   roles:
-    - { role: clone-repository, project: "opnfv/bifrost", repo: "{{ OPENSTACK_BIFROST_GIT_URL }}", dest: "{{ OPENSTACK_BIFROST_PATH }}", version: "{{ OPENSTACK_BIFROST_VERSION }}" }
+    - role: clone-repository
+      project: "opnfv/bifrost"
+      repo: "{{ OPENSTACK_BIFROST_GIT_URL }}"
+      dest: "{{ XCI_CACHE }}/repos/bifrost"
+      version: "{{ OPENSTACK_BIFROST_VERSION }}"
 
   tasks:
     - name: Load distribution variables
@@ -27,7 +31,7 @@
       # command module is much faster than the copy module
       synchronize:
         src: "{{ OPENSTACK_BIFROST_DEV_PATH }}"
-        dest: "{{ OPENSTACK_BIFROST_PATH }}"
+        dest: "{{ XCI_CACHE }}/repos/bifrost"
         recursive: yes
         delete: yes
       when:
@@ -35,4 +39,4 @@
     - name: combine opnfv/releng-xci and openstack/bifrost scripts/playbooks
       copy:
         src: "{{ XCI_PATH}}/bifrost/"
-        dest: "{{ OPENSTACK_BIFROST_PATH }}"
+        dest: "{{ XCI_CACHE }}/repos/bifrost"

diff --git a/xci/var/opnfv.yml b/xci/var/opnfv.yml
index f13a94e..19a5cca 100644 (file)
--- a/xci/var/opnfv.yml
+++ b/xci/var/opnfv.yml
@@ -10,7 +10,6 @@
 OPNFV_RELENG_GIT_URL: "{{ lookup('env','OPNFV_RELENG_GIT_URL') }}"
 OPNFV_RELENG_VERSION: "{{ lookup('env','OPNFV_RELENG_VERSION') }}"
 OPENSTACK_BIFROST_GIT_URL: "{{ lookup('env','OPENSTACK_BIFROST_GIT_URL') }}"
-OPENSTACK_BIFROST_PATH: "{{ lookup('env','OPENSTACK_BIFROST_PATH') }}"
 OPENSTACK_BIFROST_DEV_PATH: "{{ lookup('env','OPENSTACK_BIFROST_DEV_PATH') }}"
 OPENSTACK_BIFROST_VERSION: "{{ lookup('env','OPENSTACK_BIFROST_VERSION') }}"
 OPENSTACK_OSA_GIT_URL: "{{ lookup('env','OPENSTACK_OSA_GIT_URL') }}"

diff --git a/xci/xci-deploy.sh b/xci/xci-deploy.sh
index 0223d7d..abb648d 100755 (executable)
--- a/xci/xci-deploy.sh
+++ b/xci/xci-deploy.sh
@@ -181,7 +181,7 @@ cd $XCI_PATH/bifrost/
 sudo -E bash ./scripts/destroy-env.sh
 cd $XCI_PLAYBOOKS
 ansible-playbook ${XCI_ANSIBLE_VERBOSITY} -i inventory provision-vm-nodes.yml
-cd ${OPENSTACK_BIFROST_PATH}
+cd ${XCI_CACHE}/repos/bifrost
 bash ./scripts/bifrost-provision.sh
 echo "-----------------------------------------------------------------------"
 echo "Info: VM nodes are provisioned!"
@@ -250,7 +250,7 @@ fi
 echo "Info: Setting up target hosts for openstack-ansible"
 echo "-----------------------------------------------------------------------"
 ssh root@$OPNFV_HOST_IP "openstack-ansible ${XCI_ANSIBLE_VERBOSITY} \
-     $OPENSTACK_OSA_PATH/playbooks/setup-hosts.yml | tee setup-hosts.log "
+     releng-xci/.cache/repos/openstack-ansible/playbooks/setup-hosts.yml | tee setup-hosts.log "
 scp root@$OPNFV_HOST_IP:~/setup-hosts.log $LOG_PATH/setup-hosts.log
 echo "-----------------------------------------------------------------------"
 echo "Info: Set up target hosts for openstack-ansible successfuly"
@@ -270,7 +270,7 @@ echo "Info: Set up target hosts for openstack-ansible successfuly"
 #-------------------------------------------------------------------------------
 echo "Info: Gathering facts"
 echo "-----------------------------------------------------------------------"
-ssh root@$OPNFV_HOST_IP "cd $OPENSTACK_OSA_PATH/playbooks; \
+ssh root@$OPNFV_HOST_IP "cd releng-xci/.cache/repos/openstack-ansible/playbooks; \
         ansible ${XCI_ANSIBLE_VERBOSITY} -m setup -a gather_subset=network,hardware,virtual all"
 echo "-----------------------------------------------------------------------"
 
@@ -283,7 +283,7 @@ echo "Info: Setting up infrastructure"
 echo "-----------------------------------------------------------------------"
 echo "xci: running ansible playbook setup-infrastructure.yml"
 ssh root@$OPNFV_HOST_IP "openstack-ansible ${XCI_ANSIBLE_VERBOSITY} \
-     $OPENSTACK_OSA_PATH/playbooks/setup-infrastructure.yml | tee setup-infrastructure.log"
+     releng-xci/.cache/repos/openstack-ansible/playbooks/setup-infrastructure.yml | tee setup-infrastructure.log"
 scp root@$OPNFV_HOST_IP:~/setup-infrastructure.log $LOG_PATH/setup-infrastructure.log
 echo "-----------------------------------------------------------------------"
 # check the log to see if we have any error
@@ -300,12 +300,12 @@ echo "-----------------------------------------------------------------------"
 # Apply SUSE fix until https://review.openstack.org/508154 is merged
 if [[ ${OS_FAMILY,,} == "suse" ]]; then
        ssh root@$OPNFV_HOST_IP "ansible --ssh-extra-args='-o StrictHostKeyChecking=no' \
-               -i $OPENSTACK_OSA_PATH/playbooks/inventory/ galera_container -m shell \
+               -i releng-xci/.cache/repos/openstack-ansible/playbooks/inventory/ galera_container -m shell \
                -a \"sed -i \\\"s@/var/run/mysqld/mysqld.sock@/var/run/mysql/mysql.sock@\\\" /etc/my.cnf\""
 fi
 
 ssh root@$OPNFV_HOST_IP "ansible --ssh-extra-args='-o StrictHostKeyChecking=no' \
-    -i $OPENSTACK_OSA_PATH/playbooks/inventory/ galera_container -m shell \
+    -i releng-xci/.cache/repos/openstack-ansible/playbooks/inventory/ galera_container -m shell \
        -a \"mysql -h localhost -e \\\"show status like '%wsrep_cluster_%';\\\"\" | tee galera.log"
 scp root@$OPNFV_HOST_IP:~/galera.log $LOG_PATH/galera.log
 echo "-----------------------------------------------------------------------"
@@ -324,7 +324,7 @@ echo "Info: Database cluster verification successful!"
 echo "Info: Installing OpenStack on target hosts"
 echo "-----------------------------------------------------------------------"
 ssh root@$OPNFV_HOST_IP "openstack-ansible ${XCI_ANSIBLE_VERBOSITY} \
-     $OPENSTACK_OSA_PATH/playbooks/setup-openstack.yml | tee opnfv-setup-openstack.log"
+     releng-xci/.cache/repos/openstack-ansible/playbooks/setup-openstack.yml | tee opnfv-setup-openstack.log"
 scp root@$OPNFV_HOST_IP:~/opnfv-setup-openstack.log $LOG_PATH/opnfv-setup-openstack.log
 echo "-----------------------------------------------------------------------"
 # check the log to see if we have any error