gerrit.opnfv Code Review - calipso.git/blob - app/install/calipso-installer.py

1 ###############################################################################

3 # and others #

4 # #

6 # are made available under the terms of the Apache License, Version 2.0 #

7 # which accompanies this distribution, and is available at #

8 # http://www.apache.org/licenses/LICENSE-2.0 #

9 ###############################################################################

10 from pymongo import MongoClient, ReturnDocument

11 from pymongo.errors import ConnectionFailure

12 from urllib.parse import quote_plus

13 import docker

14 import argparse

15 import dockerpycreds

16 # note : not used, useful for docker api security if used

17 import time

18 import json

19

20 calipso_volume = {'/home/calipso': {'bind': '/local_dir', 'mode': 'rw'}}

21

22

23 class MongoComm:

24 # deals with communication from host/installer server to mongoDB,

25 # includes methods for future use

26 try:

27

28 def __init__(self, host, user, pwd, port):

29 self.uri = "mongodb://%s:%s@%s:%s/%s" % (

30 quote_plus(user), quote_plus(pwd), host, port, "calipso")

31 self.client = MongoClient(self.uri)

32

33 def find(self, coll, key, val):

34 collection = self.client.calipso[coll]

35 doc = collection.find({key: val})

36 return doc

37

38 def get(self, coll, doc_name):

39 collection = self.client.calipso[coll]

40 doc = collection.find_one({"name": doc_name})

41 return doc

42

43 def insert(self, coll, doc):

44 collection = self.client.calipso[coll]

45 doc_id = collection.insert(doc)

46 return doc_id

47

48 def remove_doc(self, coll, doc):

49 collection = self.client.calipso[coll]

50 collection.remove(doc)

51

52 def remove_coll(self, coll):

53 collection = self.client.calipso[coll]

54 collection.remove()

55

56 def find_update(self, coll, key, val, data):

57 collection = self.client.calipso[coll]

58 collection.find_one_and_update(

59 {key: val},

60 {"$set": data},

61 upsert=True

62 )

63

64 def update(self, coll, doc, upsert=False):

65 collection = self.client.calipso[coll]

66 doc_id = collection.update_one({'_id': doc['_id']},{'$set': doc},

67 upsert=upsert)

68 return doc_id

69

70 except ConnectionFailure:

71 print("MongoDB Server not available")

72

73

74 # using local host docker environment parameters

75 DockerClient = docker.from_env()

76

77 # use the below example for installer against a remote docker host:

78 # DockerClient = \

79 # docker.DockerClient(base_url='tcp://korlev-calipso-testing.cisco.com:2375')

80

81 def copy_file(filename):

82 c = MongoComm(args.hostname, args.dbuser, args.dbpassword, args.dbport)

83 txt = open('db/'+filename+'.json')

84 data = json.load(txt)

85 c.remove_coll(filename)

86 doc_id = c.insert(filename, data)

87 print("Copied", filename, "mongo doc_ids:\n\n", doc_id, "\n\n")

88 time.sleep(1)

89

90 C_MONGO_CONFIG = "/local_dir/calipso_mongo_access.conf"

91 H_MONGO_CONFIG = "/home/calipso/calipso_mongo_access.conf"

92 PYTHONPATH = "/home/scan/calipso_prod/app"

93 C_LDAP_CONFIG = "/local_dir/ldap.conf"

94 H_LDAP_CONFIG = "/home/calipso/ldap.conf"

95

96 def container_started(name: str, print_message=True):

97 found = DockerClient.containers.list(all=True, filters={"name": name})

98 if found and print_message:

99 print("container named {} already exists, "

100 "please deal with it using docker...\n"

101 .format(name))

102 return bool(found)

103

104 # functions to check and start calipso containers:

105 def start_mongo(dbport):

106 name = "calipso-mongo"

107 if container_started(name):

108 return

109 print("\nstarting container {}, please wait...\n".format(name))

110 image = DockerClient.images.list(all=True,

111 name="korenlev/calipso:mongo")

112 if image:

113 print(image, "exists...not downloading...")

114 else:

115 print("image korenlev/calipso:mongo missing, "

116 "hold on while downloading first...\n")

117 image = DockerClient.images.pull("korenlev/calipso:mongo")

118 print("Downloaded", image, "\n\n")

119 DockerClient.containers.run('korenlev/calipso:mongo',

120 detach=True,

121 name=name,

122 ports={'27017/tcp': dbport, '28017/tcp': 28017},

123 restart_policy={"Name": "always"})

124 # wait a bit till mongoDB is up before starting to copy the json files

125 # from 'db' folder:

126 time.sleep(5)

127 enable_copy = input("create initial calipso DB ? "

128 "(copy json files from 'db' folder to mongoDB - "

129 "'c' to copy, 'q' to skip):")

130 if enable_copy != "c":

131 return

132 print("\nstarting to copy json files to mongoDB...\n\n")

133 print("-----------------------------------------\n\n")

134 time.sleep(1)

135 copy_file("attributes_for_hover_on_data")

136 copy_file("clique_constraints")

137 copy_file("clique_types")

138 copy_file("cliques")

139 copy_file("constants")

140 copy_file("environments_config")

141 copy_file("inventory")

142 copy_file("link_types")

143 copy_file("links")

144 copy_file("messages")

145 copy_file("meteor_accounts_loginServiceConfiguration")

146 copy_file("users")

147 copy_file("monitoring_config")

148 copy_file("monitoring_config_templates")

149 copy_file("network_agent_types")

150 copy_file("roles")

151 copy_file("scans")

152 copy_file("scheduled_scans")

153 copy_file("statistics")

154 copy_file("supported_environments")

155

156 # note : 'messages', 'roles', 'users' and some of the 'constants'

157 # are filled by calipso-ui at runtime

158 # some other docs are filled later by scanning, logging

159 # and monitoring

160

161 def start_listen():

162 name = "calipso-listen"

163 if container_started(name):

164 return

165 print("\nstarting container {}...\n".format(name))

166 image = DockerClient.images.list(all=True,

167 name="korenlev/calipso:listen")

168 if image:

169 print(image, "exists...not downloading...")

170 else:

171 print("image korenlev/calipso:listen missing, "

172 "hold on while downloading first...\n")

173 image = DockerClient.images.pull("korenlev/calipso:listen")

174 print("Downloaded", image, "\n\n")

175 listencontainer = DockerClient.containers.run('korenlev/calipso:listen',

176 detach=True,

177 name=name,

178 ports={'22/tcp': 50022},

179 restart_policy={"Name": "always"},

180 environment=["PYTHONPATH=" + PYTHONPATH,

181 "MONGO_CONFIG=" + C_MONGO_CONFIG],

182 volumes=calipso_volume)

183

184 def start_ldap():

185 name = "calipso-ldap"

186 if container_started(name):

187 return

188 print("\nstarting container {}...\n".format(name))

189 image = DockerClient.images.list(all=True,

190 name="korenlev/calipso:ldap")

191 if image:

192 print(image, "exists...not downloading...")

193 else:

194 print("image korenlev/calipso:ldap missing, "

195 "hold on while downloading first...\n")

196 image = DockerClient.images.pull("korenlev/calipso:ldap")

197 print("Downloaded", image, "\n\n")

198 ldapcontainer = DockerClient.containers.run('korenlev/calipso:ldap',

199 detach=True,

200 name=name,

201 ports={'389/tcp': 389, '389/udp': 389},

202 restart_policy={"Name": "always"},

203 volumes=calipso_volume)

204

205 def start_api():

206 name = "calipso-api"

207 if container_started(name):

208 return

209 print("\nstarting container {}...\n".format(name))

210 image = DockerClient.images.list(all=True,

211 name="korenlev/calipso:api")

212 if image:

213 print(image, "exists...not downloading...")

214 else:

215 print("image korenlev/calipso:api missing,"

216 " hold on while downloading first...\n")

217 image = DockerClient.images.pull("korenlev/calipso:api")

218 print("Downloaded", image, "\n\n")

219 apicontainer = DockerClient.containers.run('korenlev/calipso:api',

220 detach=True,

221 name=name,

222 ports={'8000/tcp': 8000, '22/tcp': 40022},

223 restart_policy={"Name": "always"},

224 environment=["PYTHONPATH=" + PYTHONPATH,

225 "MONGO_CONFIG=" + C_MONGO_CONFIG,

226 "LDAP_CONFIG=" + C_LDAP_CONFIG,

227 "LOG_LEVEL=DEBUG"],

228 volumes=calipso_volume)

229

230 def start_scan():

231 name = "calipso-scan"

232 if container_started(name):

233 return

234 print("\nstarting container {}...\n".format(name))

235 image = DockerClient.images.list(all=True,

236 name="korenlev/calipso:scan")

237 if image:

238 print(image, "exists...not downloading...")

239 else:

240 print("image korenlev/calipso:scan missing, "

241 "hold on while downloading first...\n")

242 image = DockerClient.images.pull("korenlev/calipso:scan")

243 print("Downloaded", image, "\n\n")

244 scancontainer = DockerClient.containers.run('korenlev/calipso:scan',

245 detach=True,

246 name=name,

247 ports={'22/tcp': 30022},

248 restart_policy={"Name": "always"},

249 environment=["PYTHONPATH=" + PYTHONPATH,

250 "MONGO_CONFIG=" + C_MONGO_CONFIG],

251 volumes=calipso_volume)

252

253 def start_sensu():

254 name = "calipso-sensu"

255 if container_started(name):

256 return

257 print("\nstarting container {}...\n".format(name))

258 image = DockerClient.images.list(all=True,

259 name="korenlev/calipso:sensu")

260 if image:

261 print(image, "exists...not downloading...")

262 else:

263 print("image korenlev/calipso:sensu missing,"

264 " hold on while downloading first...\n")

265 image = DockerClient.images.pull("korenlev/calipso:sensu")

266 print("Downloaded", image, "\n\n")

267 sensucontainer = DockerClient.containers.run('korenlev/calipso:sensu',

268 detach=True,

269 name=name,

270 ports={'22/tcp': 20022, '3000/tcp': 3000, '4567/tcp': 4567,

271 '5671/tcp': 5671, '15672/tcp': 15672},

272 restart_policy={"Name": "always"},

273 environment=["PYTHONPATH=" + PYTHONPATH],

274 volumes=calipso_volume)

275

276 def start_ui(host, dbuser, dbpassword, webport, dbport):

277 name = "calipso-ui"

278 if container_started(name):

279 return

280 print("\nstarting container {}...\n".format(name))

281 image = DockerClient.images.list(all=True, name="korenlev/calipso:ui")

282 if image:

283 print(image, "exists...not downloading...")

284 else:

285 print("image korenlev/calipso:ui missing, "

286 "hold on while downloading first...\n")

287 image = DockerClient.images.pull("korenlev/calipso:ui")

288 print("Downloaded", image, "\n\n")

289 uicontainer = DockerClient.containers.run('korenlev/calipso:ui',

290 detach=True,

291 name=name,

292 ports={'3000/tcp': webport},

293 restart_policy={"Name": "always"},

294 environment=["ROOT_URL=http://{}:{}".format(host, str(webport)),

295 "MONGO_URL=mongodb://{}:{}@{}:{}/calipso".format(

296 dbuser, dbpassword, host, str(dbport)),

297 "LDAP_CONFIG=" + C_LDAP_CONFIG])

298

299 # check and stop a calipso container by given name

300 def container_stop(container_name):

301 if not container_started(container_name, print_message=False):

302 print("no container named", container_name, "found...")

303 return

304 print("fetching container name", container_name, "...\n")

305 c = DockerClient.containers.get(container_name)

306 if c.status != "running":

307 print(container_name, "is not running...")

308 else:

309 print("killing container name", c.name, "...\n")

310 c.kill()

311 time.sleep(1)

312 print("removing container name", c.name, "...\n")

313 c.remove()

314

315 # parser for getting optional command arguments:

316 parser = argparse.ArgumentParser()

317 parser.add_argument("--hostname",

318 help="Hostname or IP address of the server "

319 "(default=172.17.0.1)",

320 type=str,

321 default="172.17.0.1",

322 required=False)

323 parser.add_argument("--webport",

324 help="Port for the Calipso WebUI "

325 "(default=80)",

326 type=int,

327 default="80",

328 required=False)

329 parser.add_argument("--dbport",

330 help="Port for the Calipso MongoDB"

331 "(default=27017)",

332 type=int,

333 default="27017",

334 required=False)

335 parser.add_argument("--dbuser",

336 help="User for the Calipso MongoDB "

337 "(default=calipso)",

338 type=str,

339 default="calipso",

340 required=False)

341 parser.add_argument("--dbpassword",

342 help="Password for the Calipso MongoDB "

343 "(default=calipso_default)",

344 type=str,

345 default="calipso_default",

346 required=False)

347 args = parser.parse_args()

348

349 container = ""

350 action = ""

351 container_names = ["calipso-mongo", "calipso-scan", "calipso-listen",

352 "calipso-ldap", "calipso-api", "calipso-sensu", "calipso-ui"]

353 container_actions = ["stop", "start"]

354 while action not in container_actions:

355 action = input("Action? (stop, start, or 'q' to quit):\n")

356 if action == "q":

357 exit()

358 while container != "all" and container not in container_names:

359 container = input("Container? (all, {} or 'q' to quit):\n"

360 .format(", ".join(container_names)))

361 if container == "q":

362 exit()

363

364 # starting the containers per arguments:

365 if action == "start":

366 # building /home/calipso/calipso_mongo_access.conf and

367 # /home/calipso/ldap.conf files, per the arguments:

368 calipso_mongo_access_text =\

369 "server " + args.hostname +\

370 "\nuser " + args.dbuser +\

371 "\npwd " + args.dbpassword +\

372 "\nauth_db calipso"

373 ldap_text =\

374 "user admin" +\

375 "\npassword password" +\

376 "\nurl ldap://" + args.hostname + ":389" +\

377 "\nuser_id_attribute CN" + "\nuser_pass_attribute userpassword" +\

378 "\nuser_objectclass inetOrgPerson" +\

379 "\nuser_tree_dn OU=Users,DC=openstack,DC=org" + "\nquery_scope one" +\

380 "\ntls_req_cert allow" +\

381 "\ngroup_member_attribute member"

382 print("creating default", H_MONGO_CONFIG, "file...\n")

383 calipso_mongo_access_file = open(H_MONGO_CONFIG, "w+")

384 time.sleep(1)

385 calipso_mongo_access_file.write(calipso_mongo_access_text)

386 calipso_mongo_access_file.close()

387 print("creating default", H_LDAP_CONFIG, "file...\n")

388 ldap_file = open(H_LDAP_CONFIG, "w+")

389 time.sleep(1)

390 ldap_file.write(ldap_text)

391 ldap_file.close()

392

393 if container == "calipso-mongo" or container == "all":

394 start_mongo(args.dbport)

395 time.sleep(1)

396 if container == "calipso-listen" or container == "all":

397 start_listen()

398 time.sleep(1)

399 if container == "calipso-ldap" or container == "all":

400 start_ldap()

401 time.sleep(1)

402 if container == "calipso-api" or container == "all":

403 start_api()

404 time.sleep(1)

405 if container == "calipso-scan" or container == "all":

406 start_scan()

407 time.sleep(1)

408 if container == "calipso-sensu" or container == "all":

409 start_sensu()

410 time.sleep(1)

411 if container == "calipso-ui" or container == "all":

412 start_ui(args.hostname, args.dbuser, args.dbpassword, args.webport,

413 args.dbport)

414 time.sleep(1)

415

416 # stopping the containers per arguments:

417 if action == "stop":

418 for name_to_stop in container_names:

419 if container == name_to_stop or container == "all":

420 container_stop(name_to_stop)