docker/smoke/testcases.yaml

   1 ---
   2 tiers:
   3   - name: smoke
   4     description: >-
   5       A set of conformance tests integrated from kubernetes project.
   6     testcases:
   7       - case_name: xrally_kubernetes
   8         project_name: functest
   9         criteria: 100
  10         blocking: false
  11         description: >-
  12           All in one tasks for checking basic functionality of
  13           Kubernetes cluster.
  14         dependencies:
  15           - DEPLOY_SCENARIO: "k8-*"
  16         run:
  17           name: xrally_kubernetes
  18       - case_name: k8s_conformance
  19         project_name: functest
  20         criteria: 100
  21         blocking: false
  22         description: >-
  23           Conformance Tests are expect to pass on any Kubernetes
  24           cluster, this tests are necessary demonstrate conformance
  25           of the Kubernetes deployment.
  26         dependencies:
  27           - DEPLOY_SCENARIO: "k8-*"
  28         run:
  29           name: e2e_testing
  30           args:
  31             nodes: 10
  32             ginkgo:
  33               focus: \[Conformance\]
  34               skip: \[Serial\]|\[Disruptive\]|NoExecuteTaintManager
  35       - case_name: k8s_conformance_serial
  36         project_name: functest
  37         criteria: 100
  38         blocking: false
  39         description: >-
  40           Conformance Tests are expect to pass on any Kubernetes
  41           cluster, this tests are necessary demonstrate conformance
  42           of the Kubernetes deployment.
  43         dependencies:
  44           - DEPLOY_SCENARIO: "k8-*"
  45         run:
  46           name: e2e_testing
  47           args:
  48             ginkgo:
  49               focus: \[Serial\].*\[Conformance\]
  50               skip: \[Disruptive\]|NoExecuteTaintManager
  51       - case_name: sig_api_machinery
  52         project_name: functest
  53         criteria: 100
  54         blocking: false
  55         description: >-
  56           It runs the remaining sig-api-machinery testing excluding
  57           the label Conformance (part of k8s_conformance) and
  58           StorageVersionAPI.
  59         dependencies:
  60           - DEPLOY_SCENARIO: "k8-*"
  61         run:
  62           name: e2e_testing
  63           args:
  64             nodes: 10
  65             ginkgo:
  66               focus: \[sig-api-machinery\]
  67               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
  68                 \\[Serial\\]|\\[Conformance\\]|\
  69                 \\[Feature:StorageVersionAPI\\]"
  70       - case_name: sig_api_machinery_serial
  71         project_name: functest
  72         criteria: 100
  73         blocking: false
  74         description: >-
  75           It runs the remaining sig-api-machinery testing excluding
  76           the label Conformance (part of k8s_conformance) and
  77           StorageVersionAPI.
  78         dependencies:
  79           - DEPLOY_SCENARIO: "k8-*"
  80         run:
  81           name: e2e_testing
  82           args:
  83             ginkgo:
  84               focus: \[sig-api-machinery\].*\[Serial\]
  85               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
  86                 \\[Conformance\\]|\
  87                 \\[Feature:StorageVersionAPI\\]"
  88       - case_name: sig_apps
  89         project_name: functest
  90         criteria: 100
  91         blocking: false
  92         description: >-
  93           It runs the remaining sig-apps testing excluding the
  94           following labels: Conformance (part of k8s_conformance) and
  95           both StatefulUpgrade and StatefulSet features.
  96         dependencies:
  97           - DEPLOY_SCENARIO: "k8-*"
  98         run:
  99           name: e2e_testing
 100           args:
 101             nodes: 10
 102             ginkgo:
 103               focus: \[sig-apps]
 104               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 105                 \\[Serial\\]|\\[Conformance\\]|\
 106                 \\[Feature:IndexedJob\\]|\
 107                 \\[Feature:SuspendJob\\]|\
 108                 \\[Feature:StatefulUpgrade\\]|\
 109                 \\[Feature:StatefulSet\\]"
 110       - case_name: sig_apps_serial
 111         project_name: functest
 112         criteria: 100
 113         blocking: false
 114         description: >-
 115           It runs the remaining sig-apps testing excluding the
 116           following labels: Conformance (part of k8s_conformance) and
 117           both StatefulUpgrade and StatefulSet features.
 118         dependencies:
 119           - DEPLOY_SCENARIO: "k8-*"
 120         run:
 121           name: e2e_testing
 122           args:
 123             ginkgo:
 124               focus: \[sig-apps].*\[Serial\]
 125               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 126                 \\[Conformance\\]|\
 127                 \\[Feature:StatefulUpgrade\\]|\
 128                 \\[Feature:DaemonSetUpdateSurge\\]|\
 129                 \\[Feature:StatefulSet\\]"
 130       - case_name: sig_auth
 131         project_name: functest
 132         criteria: 100
 133         blocking: false
 134         description: >-
 135           It runs the remaining sig-auth testing excluding the
 136           following labels: Conformance (part of k8s_conformance) and
 137           both PodSecurityPolicy and BoundServiceAccountTokenVolume
 138           features.
 139         dependencies:
 140           - DEPLOY_SCENARIO: "k8-*"
 141         run:
 142           name: e2e_testing
 143           args:
 144             nodes: 10
 145             ginkgo:
 146               focus: \[sig-auth\]
 147               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 148                 \\[Serial\\]|\\[Conformance\\]|\
 149                 \\[Feature:PodSecurityPolicy\\]|\
 150                 \\[Feature:BoundServiceAccountTokenVolume\\]"
 151       - case_name: sig_cli
 152         project_name: functest
 153         criteria: 100
 154         blocking: false
 155         description: >-
 156           It runs the remaining sig-cli testing excluding Conformance
 157           (part of k8s_conformance).
 158         dependencies:
 159           - DEPLOY_SCENARIO: "k8-*"
 160         run:
 161           name: e2e_testing
 162           args:
 163             nodes: 10
 164             ginkgo:
 165               focus: \[sig-cli]
 166               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 167                 \\[Serial\\]|\\[Conformance\\]"
 168       - case_name: sig_cli_serial
 169         project_name: functest
 170         criteria: 100
 171         blocking: false
 172         description: >-
 173           It runs the remaining sig-cli testing excluding Conformance
 174           (part of k8s_conformance).
 175         dependencies:
 176           - DEPLOY_SCENARIO: "k8-*"
 177         run:
 178           name: e2e_testing
 179           args:
 180             ginkgo:
 181               focus: \[sig-cli].*\[Serial\]
 182               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 183                 \\[Conformance\\]"
 184       - case_name: sig_cluster_lifecycle
 185         project_name: functest
 186         criteria: 100
 187         blocking: false
 188         description: >-
 189           It runs the remaining sig-cluster-lifecycle testing
 190           excluding Conformance (part of k8s_conformance).
 191         dependencies:
 192           - DEPLOY_SCENARIO: "k8-*"
 193         run:
 194           name: e2e_testing
 195           args:
 196             nodes: 10
 197             ginkgo:
 198               focus: \[sig-cluster-lifecycle\]
 199               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 200                 \\[Serial\\]|\\[Conformance\\]"
 201       - case_name: sig_instrumentation
 202         project_name: functest
 203         criteria: 100
 204         blocking: false
 205         description: >-
 206           It runs the remaining sig-instrumentation testing excluding
 207           Conformance (part of k8s_conformance)
 208         dependencies:
 209           - DEPLOY_SCENARIO: "k8-*"
 210         run:
 211           name: e2e_testing
 212           args:
 213             nodes: 10
 214             ginkgo:
 215               focus: \[sig-instrumentation\]
 216               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 217                 \\[Serial\\]|\\[Conformance\\]"
 218       - case_name: sig_network
 219         project_name: functest
 220         criteria: 100
 221         blocking: false
 222         description: >-
 223           It runs the remaining sig-network testing excluding the
 224           following labels: Conformance (part of k8s_conformance),
 225           load.balancer, NetworkPolicy, Networking-IPv6,
 226           IPv6DualStackAlphaFeature, SCTP and Example. A couple of features may
 227           be automatically skipped like NEG and kubemci.
 228         dependencies:
 229           - DEPLOY_SCENARIO: "k8-*"
 230         run:
 231           name: e2e_testing
 232           args:
 233             nodes: 10
 234             ginkgo:
 235               focus: \[sig-network\]
 236               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 237                 \\[Serial\\]|\\[Conformance\\]|load.balancer|\
 238                 LoadBalancer|\\[Feature:NetworkPolicy\\]|\
 239                 \\[Feature:Networking-IPv6\\]|\
 240                 \\[Feature:IPv6DualStack\\]|\
 241                 \\[Feature:SCTP\\]|\\[Feature:Example\\]|\
 242                 Network.should.set.TCP.CLOSE_WAIT.timeout"
 243       - case_name: sig_network_serial
 244         project_name: functest
 245         criteria: 100
 246         blocking: false
 247         description: >-
 248           It runs the remaining sig-network testing excluding the
 249           following labels: Conformance (part of k8s_conformance),
 250           load.balancer, NetworkPolicy, Networking-IPv6,
 251           IPv6DualStackAlphaFeature, SCTP and Example. A couple of features may
 252           be automatically skipped like NEG and kubemci.
 253         dependencies:
 254           - DEPLOY_SCENARIO: "k8-*"
 255         run:
 256           name: e2e_testing
 257           args:
 258             ginkgo:
 259               focus: \[sig-network\].*\[Serial\]
 260               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 261                 \\[Conformance\\]|load.balancer|\
 262                 LoadBalancer|\\[Feature:NetworkPolicy\\]|\
 263                 \\[Feature:SCTP\\]|\\[Feature:Example\\]"
 264       - case_name: sig_node
 265         project_name: functest
 266         criteria: 100
 267         blocking: false
 268         description: >-
 269           It runs the remaining sig-node testing excluding the
 270           following labels: Conformance (part of k8s_conformance),
 271           RuntimeClass, all RuntimeHandler, PodGarbageCollector,
 272           RegularResourceUsageTrackingfeatures and
 273           ExperimentalResourceUsageTracking features.
 274         dependencies:
 275           - DEPLOY_SCENARIO: "k8-*"
 276         run:
 277           name: e2e_testing
 278           args:
 279             nodes: 10
 280             ginkgo:
 281               focus: \[sig-node]
 282               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 283                 \\[Serial\\]|\\[Conformance\\]|\
 284                 \\[NodeFeature:RuntimeHandler\\]|\
 285                 \\[Feature:PodGarbageCollector\\]|\
 286                 \\[Feature:RegularResourceUsageTracking\\]|\
 287                 \\[Feature:GPUUpgrade\\]|\
 288                 \\[Feature:ExperimentalResourceUsageTracking\\]|\
 289                 RuntimeClass"
 290       - case_name: sig_scheduling_serial
 291         project_name: functest
 292         criteria: 100
 293         blocking: false
 294         description: >-
 295           It runs the remaining sig-network testing excluding the
 296           next labels: Conformance (part of k8s_conformance) and the
 297           GPUDevicePlugin feature.
 298         dependencies:
 299           - DEPLOY_SCENARIO: 'k8-*'
 300         run:
 301           name: e2e_testing
 302           args:
 303             ginkgo:
 304               focus: \[sig-scheduling\].*\[Serial\]
 305               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 306                 \\[Conformance\\]|\
 307                 \\[Feature:GPUDevicePlugin\\]"
 308       - case_name: sig_storage
 309         project_name: functest
 310         criteria: 100
 311         blocking: false
 312         description: >-
 313           It runs the remaining sig-storage testing excluding the
 314           following labels: Conformance (part of k8s_conformance),
 315           all features, all drivers, PersistentVolumes.NFS and Slow.
 316         dependencies:
 317           - DEPLOY_SCENARIO: "k8-*"
 318         run:
 319           name: e2e_testing
 320           args:
 321             nodes: 10
 322             ginkgo:
 323               focus: \[sig-storage\]
 324               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 325                 \\[Serial\\]|\\[Conformance\\]|\
 326                 \\[Feature:.+\\]|\\[Driver:.+\\]|\
 327                 PersistentVolumes.NFS|\\[Slow\\]"
 328       - case_name: sig_storage_serial
 329         project_name: functest
 330         criteria: 100
 331         blocking: false
 332         description: >-
 333           It runs the remaining sig-storage testing excluding the
 334           following labels: Conformance (part of k8s_conformance),
 335           all features, all drivers, PersistentVolumes.NFS and Slow.
 336         dependencies:
 337           - DEPLOY_SCENARIO: "k8-*"
 338         run:
 339           name: e2e_testing
 340           args:
 341             ginkgo:
 342               focus: \[sig-storage\].*\[Serial\]
 343               skip: "\\[Disruptive\\]|\\[Flaky\\]|\\[alpha\\]|\
 344                 \\[Conformance\\]|\
 345                 \\[Feature:.+\\]|\\[Driver:.+\\]|\
 346                 PersistentVolumes.NFS|\\[Slow\\]"