From: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
Date: Fri, 2 Mar 2018 17:29:53 +0000 (+0100)
Subject: generate_config: New `-b` arg for batch processing
X-Git-Url: https://gerrit.opnfv.org/gerrit/gitweb?a=commitdiff_plain;h=refs%2Fchanges%2F13%2F53013%2F3;p=pharos.git

generate_config: New `-b` arg for batch processing

Sometimes, parsing multiple j2 templates based on the same PDF+IDF
is needed, in which case we'd have to call generate_config once for
each template.

Add a new argument, `-b`, which allows batch processing of multiple
templates in one go:
- files ending in '.j2' (e.g. 'template.yml.j2') will be expanded to
  a file without the '.j2' suffix in the same directory as the
  source template (e.g. 'template.yml');
- templates not ending in '.j2' are skipped in batch mode;
- in order to pass multiple templates, `-j` argument can now be passed
  multiple times, similar to `-i`;
- although relative filepaths still work, it is highly recommended to
  use the full path (abspath) for j2 templates;
- to avoid filename collisions, j2 parser will open '/' (as well as
  any directories specified via `-i`) and lookup each j2 template
  by its full path;

Add a new argument, `-v` for verbose logging.

While at it, import only needed functions from 'os.path' to make our
code more compact and easier to read.

Change-Id: I2291935d42e4196813cd75154d5b674bcc330183
Signed-off-by: Alexandru Avadanii <Alexandru.Avadanii@enea.com>
---

diff --git a/config/utils/generate_config.py b/config/utils/generate_config.py
index f02acf5a..f45f7888 100755
--- a/config/utils/generate_config.py
+++ b/config/utils/generate_config.py
@@ -11,7 +11,7 @@
 
 import argparse
 import logging
-import os
+from os.path import abspath, exists, isfile, split
 from subprocess import CalledProcessError, check_output
 import gen_config_lib
 import yaml
@@ -20,13 +20,22 @@ from jinja2 import Environment, FileSystemLoader
 
 PARSER = argparse.ArgumentParser()
 PARSER.add_argument("--yaml", "-y", type=str, required=True)
-PARSER.add_argument("--jinja2", "-j", type=str, required=True)
-PARSER.add_argument("--includesdir", "-i", type=str, action='append')
+PARSER.add_argument("--jinja2", "-j", type=str, required=True, action='append')
+PARSER.add_argument("--includesdir", "-i", action='append', default=['/'])
+PARSER.add_argument("--batch", "-b", action='store_true')
+PARSER.add_argument("--verbose", "-v", action='count')
 ARGS = PARSER.parse_args()
+
 LOADER = yaml.CSafeLoader if yaml.__with_libyaml__ else yaml.SafeLoader
+ARGS.jinja2 = [abspath(x) for x in ARGS.jinja2]
+
+logging.basicConfig()
+LOGGER = logging.getLogger('generate_config')
+if ARGS.verbose:
+    LOGGER.setLevel(logging.INFO)
 
 ENV = Environment(
-    loader=FileSystemLoader([os.path.dirname(ARGS.jinja2)] + ARGS.includesdir),
+    loader=FileSystemLoader(ARGS.includesdir),
     extensions=['jinja2.ext.do']
 )
 gen_config_lib.load_custom_filters(ENV)
@@ -34,13 +43,13 @@ gen_config_lib.load_custom_filters(ENV)
 # Run `eyaml decrypt` on the whole file, but only if PDF data is encrypted
 # Note: eyaml return code is 0 even if keys are not available
 try:
-    if os.path.isfile(ARGS.yaml) and 'ENC[PKCS7' in open(ARGS.yaml).read():
+    if isfile(ARGS.yaml) and 'ENC[PKCS7' in open(ARGS.yaml).read():
         DICT = yaml.load(check_output(['eyaml', 'decrypt',
                                        '-f', ARGS.yaml]), Loader=LOADER)
 except CalledProcessError as ex:
-    logging.error('eyaml decryption failed! Fallback to raw data.')
+    LOGGER.error('eyaml decryption failed! Fallback to raw data.')
 except OSError as ex:
-    logging.warn('eyaml not found, skipping decryption. Fallback to raw data.')
+    LOGGER.warn('eyaml not found, skipping decryption. Fallback to raw data.')
 try:
     DICT['details']
 except (NameError, TypeError) as ex:
@@ -48,8 +57,8 @@ except (NameError, TypeError) as ex:
         DICT = yaml.load(_, Loader=LOADER)
 
 # If an installer descriptor file (IDF) exists, include it (temporary)
-IDF_PATH = '/idf-'.join(os.path.split(ARGS.yaml))
-if os.path.exists(IDF_PATH):
+IDF_PATH = '/idf-'.join(split(ARGS.yaml))
+if exists(IDF_PATH):
     with open(IDF_PATH) as _:
         IDF = yaml.load(_, Loader=LOADER)
         DICT['idf'] = IDF['idf']
@@ -57,8 +66,17 @@ if os.path.exists(IDF_PATH):
 # Print dictionary generated from yaml (uncomment for debug)
 # print(DICT)
 
-# Render template and print generated conf to console
-TEMPLATE = ENV.get_template(os.path.basename(ARGS.jinja2))
-
-# pylint: disable=superfluous-parens
-print(TEMPLATE.render(conf=DICT))
+for _j2 in ARGS.jinja2:
+    TEMPLATE = ENV.get_template(_j2)
+    OUTPUT = TEMPLATE.render(conf=DICT)
+    # Render template and write generated conf to file or stdout
+    if ARGS.batch:
+        if _j2.endswith('.j2'):
+            LOGGER.info('Parsing {}'.format(_j2))
+            with open(_j2[:-3], 'w') as _:
+                _.write(OUTPUT)
+        else:
+            LOGGER.warn('Skipping {}, name does not end in ".j2"'.format(_j2))
+    else:
+        # pylint: disable=superfluous-parens
+        print(OUTPUT)