Merge "dockerfile: update for OPNFV docker build"

diff --git a/3rd_party/collectd-ves-app/ves_app/guest.yaml b/3rd_party/collectd-ves-app/ves_app/guest.yaml
index c26bd08..0f48b5e 100644 (file)
--- a/3rd_party/collectd-ves-app/ves_app/guest.yaml
+++ b/3rd_party/collectd-ves-app/ves_app/guest.yaml
@@ -223,6 +223,7 @@ Host Measurements: !Measurements
                       plugin_instance: "{vl.plugin_instance}"
                       type: disk_io_time
                       ds_name: io_time
+                  - DEFAULT: 0
                 diskMergedReadLast: !ValueItem
                   - SELECT:
                       plugin: disk
@@ -243,12 +244,14 @@ Host Measurements: !Measurements
                       plugin_instance: "{vl.plugin_instance}"
                       type: disk_time
                       ds_name: read
+                  - DEFAULT: 0
                 diskTimeWriteLast: !ValueItem
                   - SELECT:
                       plugin: disk
                       plugin_instance: "{vl.plugin_instance}"
                       type: disk_time
                       ds_name: write
+                  - DEFAULT: 0
   - SELECT:
       plugin: memory
       type_instance: free

diff --git a/3rd_party/collectd-ves-app/ves_app/host.yaml b/3rd_party/collectd-ves-app/ves_app/hypervisor.yaml
similarity index 100%
rename from 3rd_party/collectd-ves-app/ves_app/host.yaml
rename to 3rd_party/collectd-ves-app/ves_app/hypervisor.yaml

diff --git a/3rd_party/collectd-ves-app/ves_app/node.yaml b/3rd_party/collectd-ves-app/ves_app/node.yaml
new file mode 100644 (file)
index 0000000..24f2c63
--- /dev/null
+++ b/3rd_party/collectd-ves-app/ves_app/node.yaml
@@ -0,0 +1,468 @@
+---
+# Common event header definition (required fields and defaults)
+commonEventHeader: &commonEventHeader
+  domain: N/A
+  eventId: "{system.id}"
+  eventName: ""
+  eventType: Info
+  lastEpochMicrosec: 0
+  priority: Normal
+  reportingEntityId: &reportingEntityId "{system.hostname}"
+  reportingEntityName: *reportingEntityId
+  sequence: 0
+  sourceName: N/A
+  startEpochMicrosec: 0
+  version: 2.0
+
+# Host measurements definition
+Host Measurements: !Measurements
+  - ITEM-DESC:
+      event:
+        commonEventHeader: &hostCommonEventHeader
+          <<: *commonEventHeader
+          eventType: hostOS
+          domain: measurementsForVfScaling
+          sourceId: &sourceId "{vl.host}"
+          sourceName: *sourceId
+          startEpochMicrosec: !Number "{vl.time}"
+        measurementsForVfScalingFields: &hostMeasurementsForVfScalingFields
+          measurementsForVfScalingVersion: 2.0
+          additionalMeasurements: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: "/^(?!memory|cpu|interface|disk).*$/"
+            - INDEX-KEY:
+                - host
+                - plugin
+                - plugin_instance
+            - ITEM-DESC:
+                name: !StripExtraDash "{vl.plugin}-{vl.plugin_instance}"
+                arrayOfFields: !ArrayItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: "{vl.plugin}"
+                      plugin_instance: "{vl.plugin_instance}"
+                  - ITEM-DESC:
+                      name: !StripExtraDash "{vl.type}-{vl.type_instance}-{vl.ds_name}"
+                      value: "{vl.value}"
+          measurementInterval: !Number "{vl.interval}"
+          memoryUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: memory
+                type: memory
+                type_instance: free
+            - ITEM-DESC:
+                vmIdentifier: "{vl.host}"
+                memoryFree: !Number "{vl.value}"
+                memoryUsed: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: used
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                memoryBuffered: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: buffered
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                memoryCached: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: cached
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                memorySlabRecl: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: slab_recl
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                  - DEFAULT: 0
+                memorySlabUnrecl: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: memory
+                      type: memory
+                      type_instance: slab_unrecl
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                  - DEFAULT: 0
+          cpuUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: cpu
+                type: percent
+                type_instance: idle
+            - ITEM-DESC:
+                cpuIdentifier: "{vl.plugin_instance}"
+                cpuIdle: !Number "{vl.value}"
+                percentUsage: 0.0
+                cpuUsageUser: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: user
+                cpuWait: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: wait
+                cpuUsageInterrupt: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: interrupt
+                cpuUsageNice: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: nice
+                cpuUsageSoftIrq: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: softirq
+                cpuUsageSteal: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: steal
+                cpuUsageSystem: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: cpu
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: percent
+                      type_instance: system
+          vNicPerformanceArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: interface
+                type: if_packets
+                ds_name: rx
+            - ITEM-DESC:
+                valuesAreSuspect: "true"
+                vNicIdentifier: "{vl.plugin_instance}"
+                receivedTotalPacketsAccumulated: !Number "{vl.value}"
+                transmittedTotalPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_packets
+                      ds_name: tx
+                receivedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      ds_name: rx
+                transmittedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      ds_name: tx
+                receivedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      ds_name: rx
+                transmittedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      ds_name: tx
+                receivedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      ds_name: rx
+                transmittedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: interface
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      ds_name: tx
+          diskUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: disk
+                type: disk_octets
+                ds_name: read
+            - ITEM-DESC:
+                diskIdentifier: "{vl.plugin_instance}"
+                diskOctetsReadLast: !Number "{vl.value}"
+                diskOctetsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_octets
+                      ds_name: write
+                diskOpsReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      ds_name: read
+                diskOpsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      ds_name: write
+                diskIoTimeLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_io_time
+                      ds_name: io_time
+                  - DEFAULT: 0
+                diskMergedReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_merged
+                      ds_name: read
+                  - DEFAULT: 0
+                diskMergedWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_merged
+                      ds_name: write
+                  - DEFAULT: 0
+                diskTimeReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_time
+                      ds_name: read
+                  - DEFAULT: 0
+                diskTimeWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: disk
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_time
+                      ds_name: write
+                  - DEFAULT: 0
+  - SELECT:
+      plugin: memory
+      plugin_instance: host
+      type_instance: free
+
+# Guest measurements definition
+Guest Measurements: !Measurements
+  - ITEM-DESC:
+      event:
+        commonEventHeader:
+          <<: *hostCommonEventHeader
+          eventType: guestOS
+        measurementsForVfScalingFields: *hostMeasurementsForVfScalingFields
+  - SELECT:
+      plugin: memory
+      plugin_instance: guest
+      type_instance: free
+
+# Hypervisor measurements definition
+Hypervisor Measurements: !Measurements
+  - ITEM-DESC:
+      event:
+        commonEventHeader:
+          <<: *commonEventHeader
+          eventType: hypervisor
+          domain: measurementsForVfScaling
+          sourceId: &hypervisor_sourceId "{vl.plugin_instance}"
+          sourceName: *hypervisor_sourceId
+          startEpochMicrosec: !Number "{vl.time}"
+        measurementsForVfScalingFields:
+          measurementsForVfScalingVersion: 2.0
+          additionalMeasurements: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: "/^(?!memory|virt_vcpu|disk_octets|disk_ops|if_packets|if_errors|if_octets|if_dropped).*$/"
+            - INDEX-KEY:
+                - host
+                - plugin_instance
+                - type
+            - ITEM-DESC:
+                name: "{vl.type}"
+                arrayOfFields: !ArrayItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: "{vl.type}"
+                  - ITEM-DESC:
+                      name: !StripExtraDash "{vl.type}-{vl.type_instance}-{vl.ds_name}"
+                      value: "{vl.value}"
+          measurementInterval: !Number "{vl.interval}"
+          memoryUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: memory
+                type_instance: total
+            - ITEM-DESC:
+                memoryConfigured: !Bytes2Kibibytes "{vl.value}"
+                vmIdentifier: "{vl.plugin_instance}"
+                memoryUsed: 0.0
+                memoryFree: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: memory
+                      type_instance: rss
+                  - VALUE: !Bytes2Kibibytes "{vl.value}"
+                  - DEFAULT: 0
+          cpuUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: virt_vcpu
+            - ITEM-DESC:
+                cpuIdentifier: "{vl.type_instance}"
+                percentUsage: !Number "{vl.value}"
+          vNicPerformanceArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: if_packets
+                ds_name: rx
+            - ITEM-DESC:
+                valuesAreSuspect: "true"
+                vNicIdentifier: "{vl.type_instance}"
+                receivedTotalPacketsAccumulated: !Number "{vl.value}"
+                transmittedTotalPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_packets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+                receivedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: rx
+                transmittedOctetsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_octets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+                receivedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      type_instance: "{vl.type_instance}"
+                      ds_name: rx
+                transmittedErrorPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_errors
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+                receivedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      type_instance: "{vl.type_instance}"
+                      ds_name: rx
+                transmittedDiscardedPacketsAccumulated: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: if_dropped
+                      type_instance: "{vl.type_instance}"
+                      ds_name: tx
+          diskUsageArray: !ArrayItem
+            - SELECT:
+                host: "{vl.host}"
+                plugin: virt
+                plugin_instance: "{vl.plugin_instance}"
+                type: disk_octets
+                ds_name: read
+            - ITEM-DESC:
+                diskIdentifier: "{vl.type_instance}"
+                diskOctetsReadLast: !Number "{vl.value}"
+                diskOctetsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_octets
+                      type_instance: "{vl.type_instance}"
+                      ds_name: write
+                diskOpsReadLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      type_instance: "{vl.type_instance}"
+                      ds_name: read
+                diskOpsWriteLast: !ValueItem
+                  - SELECT:
+                      host: "{vl.host}"
+                      plugin: virt
+                      plugin_instance: "{vl.plugin_instance}"
+                      type: disk_ops
+                      type_instance: "{vl.type_instance}"
+                      ds_name: write
+  - SELECT:
+      plugin: virt
+      type_instance: virt_cpu_total

diff --git a/3rd_party/collectd-ves-app/ves_app/normalizer.py b/3rd_party/collectd-ves-app/ves_app/normalizer.py
index 84de5e3..dcb02f2 100644 (file)
--- a/3rd_party/collectd-ves-app/ves_app/normalizer.py
+++ b/3rd_party/collectd-ves-app/ves_app/normalizer.py
@@ -241,7 +241,7 @@ class CollectdNotification(CollectdData):
         self.message = message
 
     def __repr__(self):
-        return '{}(host={}, plugin={}, plugin_instance={}, type={},' \
+        return '{}(host={}, plugin={}, plugin_instance={}, type={}, ' \
                'type_instance={}, severity={}, message={}, time={})'.format(
                    self.__class__.__name__, self.host, self.plugin,
                    self.plugin_instance, self.type, self.type_instance,
@@ -274,7 +274,7 @@ class CollectdValue(CollectdData):
                              self.type, self.type_instance, self.ds_name)
 
     def __repr__(self):
-        return '{}(host={}, plugin={}, plugin_instance={}, type={},' \
+        return '{}(host={}, plugin={}, plugin_instance={}, type={}, ' \
                'type_instance={}, ds_name={}, value={}, time={})'.format(
                    self.__class__.__name__, self.host, self.plugin,
                    self.plugin_instance, self.type, self.type_instance,
@@ -347,14 +347,15 @@ class ValueItem(Item):
         # select collectd metric based on SELECT condition
         metrics = loader.collector.items(select)
         assert len(metrics) < 2, \
-            'Wrong SELECT condition, selected {} metrics'.format(len(metrics))
+            'Wrong SELECT condition {}, selected {} metrics'.format(
+            select, len(metrics))
         if len(metrics) > 0:
             item = cls.format_node(value_desc, {'vl': metrics[0],
                                    'system': loader.system})
             return loader.construct_object(item)
         # nothing has been found by SELECT condition, set to DEFAULT value.
-        assert default is not None, \
-            "No metrics selected by SELECT condition and DEFAULT key isn't set"
+        assert default is not None, "No metrics selected by SELECT condition" \
+            " {} and DEFAULT key isn't set".format(select)
         return default
 
 
@@ -398,10 +399,10 @@ class ArrayItem(Item):
         if len(index_keys) > 0:
             metric_set = set()
             for metric in metrics:
-                value_params = {}
+                value = CollectdValue()
                 for key in index_keys:
-                    value_params[key] = getattr(metric, key)
-                metric_set.add(CollectdValue(**value_params))
+                    setattr(value, key, getattr(metric, key))
+                metric_set.add(value)
             metrics = list(metric_set)
         # build items based on SELECT and/or INDEX-KEY criteria
         for metric in metrics:
@@ -465,7 +466,7 @@ class StripExtraDash(yaml.YAMLObject):
 
     @classmethod
     def from_yaml(cls, loader, node):
-        return '-'.join([ x for x in node.value.split('-') if len(x) > 0])
+        return '-'.join([x for x in node.value.split('-') if len(x) > 0])
 
 
 class MapValue(yaml.YAMLObject):

diff --git a/ci/utility/intel_cmt_cat_build_install_rpm.sh b/ci/utility/intel_cmt_cat_build_install_rpm.sh
index a04d0a0..ba6bec0 100755 (executable)
--- a/ci/utility/intel_cmt_cat_build_install_rpm.sh
+++ b/ci/utility/intel_cmt_cat_build_install_rpm.sh
@@ -22,14 +22,14 @@ bash $DIR/check_git_repo.sh $CMTCAT_DIR $CMTCAT_REPO
 if [[ $? != 0 ]]
 then
        rm -rf $CMTCAT_DIR
-        git clone --branch $CMTCAT_BRANCH $CMTCAT_REPO $CMTCAT_DIR
+        git clone $CMTCAT_REPO $CMTCAT_DIR
        else
        cd $CMTCAT_DIR
        git reset HEAD --hard
         git pull
-       git checkout -f $CMTCAT_BRANCH
 fi
 cd $CMTCAT_DIR
+git checkout -b intel_rdt $CMTCAT_BRANCH
 wget https://github.com/01org/intel-cmt-cat/archive/${VERSION}.tar.gz --directory-prefix=$CMTCAT_DIR
 mv $CMTCAT_DIR/${VERSION}.tar.gz $RPM_WORKDIR/SOURCES/
 rpmbuild --define "_topdir $RPM_WORKDIR" -bb $CMTCAT_DIR/rpm/intel-cmt-cat.spec

diff --git a/ci/utility/package-list.sh b/ci/utility/package-list.sh
index 0594e5b..8cd1a7d 100755 (executable)
--- a/ci/utility/package-list.sh
+++ b/ci/utility/package-list.sh
@@ -20,7 +20,7 @@ COLLECTD_DIR=$WORKSPACE/collectd
 CMTCAT_DIR=$WORKSPACE/intel-cmt-cat
 
 CMTCAT_REPO=https://github.com/01org/intel-cmt-cat.git
-CMTCAT_BRANCH=master
+CMTCAT_BRANCH=8b27ad757e86a01bc51eafcb9e600605ff1beca4
 CMTCAT_VERSION="v1.1.0"
 
 COLLECTD_REPO=https://github.com/collectd/collectd.git