verigraph/tester/test.py

   1 #!/usr/bin/python
   2
   3 ##############################################################################
   4 # Copyright (c) 2017 Politecnico di Torino and others.
   5 #
   6 # All rights reserved. This program and the accompanying materials
   7 # are made available under the terms of the Apache License, Version 2.0
   8 # which accompanies this distribution, and is available at
   9 # http://www.apache.org/licenses/LICENSE-2.0
  10 ##############################################################################
  11
  12 from __future__ import print_function
  13 from jsonschema import validate
  14 from pprint import pprint
  15 import sys
  16 import requests
  17 from requests.exceptions import *
  18 from jsonschema.exceptions import *
  19 import json
  20 import getopt
  21 import os
  22 import subprocess
  23
  24 # Constants (change them, if appropriate)
  25 VERIGRAPH_PORT = "8080"
  26 TEST_CASES_DIR = "testcases"
  27 BASE_URL = "http://localhost:"+VERIGRAPH_PORT+"/verify/api/graphs/"
  28 SCHEMA_FILE = "testcase_schema.json"
  29
  30 # Variables
  31 success = 0
  32 run = 0
  33
  34 # Utils
  35 def eprint(toPrint):
  36     sys.stdout.flush()
  37     print(toPrint, file=sys.stderr)
  38     sys.stderr.flush()
  39
  40 # Print PYTHON ver
  41 print("PYTHON " + sys.version)
  42
  43 # Loading schema file
  44 try:
  45     schema = json.load(open(SCHEMA_FILE))
  46 except ValueError:
  47     eprint("Invalid json schema (check your "+SCHEMA_FILE+")!\nExiting.")
  48     exit(-1)
  49
  50 # Iterate over .json files contained in the TEST_CASES_DIR
  51 for i in os.listdir(TEST_CASES_DIR):
  52     if i.endswith(".json"):
  53         with open(TEST_CASES_DIR+os.path.sep+i) as data_file:
  54             try:
  55                 # Load json file (raise exception if malformed)
  56                 data = json.load(data_file)
  57
  58                 # Validate input json against schema (raise exception if invalid)
  59                 validate(data, schema)
  60
  61                 run += 1
  62                 print("Test case ID: "+str(data["id"]))
  63                 print("\tFILE NAME: "+i)
  64                 print("\tTEST NAME: "+data["name"])
  65                 print("\tTEST DESCRIPTION: "+data["description"])
  66
  67                 # POST the graph
  68                 r = requests.post(BASE_URL, json=data["graph"])
  69                 if r.status_code == 201:
  70                     graph_id = r.json()["id"]
  71                     print("\tCreated Graph has ID " + str(graph_id) + " on VeriGraph")
  72
  73                     # GET the policy verification result
  74                     policy = requests.get(BASE_URL+str(graph_id)+"/policy"+data["policy_url_parameters"])
  75
  76                     # Check the response
  77                     if policy.status_code == 200:
  78                         print("\tVerification result is " + policy.json()["result"])
  79
  80                         # Check the result with the expected one
  81                         if policy.json()["result"] == data["result"]:
  82                             # SUCCESS
  83                             print("\t+++ Test passed +++")
  84                             success += 1
  85                         else:
  86                             # FAIL
  87                             eprint("\t[ERROR] Expected result was " + data["result"] + " but VeriGraph returned " + policy.json()["result"])
  88                             print("\t--- Test failed ---")
  89                     else:
  90                         print("\tVeriGraph returned an unexpected response -> " + str(policy.status_code), policy.reason)
  91                         print("\t--- Test failed ---")
  92                 print()
  93             except ValueError:
  94                 print("Malformed json!\nSkipping "+i+" file")
  95                 print("\t--- Test failed ---")
  96             except ValidationError:
  97                 print("Invalid json (see Schema file)!\nSkipping "+i+" file")
  98                 print("\t--- Test failed ---")
  99             except ConnectionError:
 100                 print("Connection refused!")
 101                 print("\t--- Test failed ---")
 102             except HTTPError:
 103                 print("HTTP error!")
 104                 print("\t--- Test failed ---")
 105
 106 # Final output
 107 print("\nTest run = "+str(run))
 108 print("Test succeded = "+str(success))
 109 if run != 0:
 110     if success != run:
 111         print("\n --- Some tests failed. See the output. ---")
 112     else:
 113         print("\n +++ All tests passed +++")
 114 else:
 115     print("\n\n +++ 0 tests executed +++")