From 3e90898f667afa508137e7be885daa62fbdb86d9 Mon Sep 17 00:00:00 2001 From: Jing Sun Date: Mon, 6 Mar 2017 17:11:21 +0800 Subject: [PATCH] Jira ESCALATOR-41:get cluster list from installer Change-Id: Ie3cd22b2f8398ec893686445ac85da7e69ffffb7 Signed-off-by: Jing Sun --- api/escalator/api/v1/clusters.py | 123 ++++++++++++++++ api/escalator/api/v1/controller.py | 13 ++ api/escalator/api/v1/router.py | 7 + api/escalator/installer/__init__.py | 0 api/escalator/installer/daisy/__init__.py | 0 api/escalator/installer/daisy/api.py | 14 ++ client/escalatorclient/v1/client.py | 2 + client/escalatorclient/v1/clusters.py | 234 ++++++++++++++++++++++++++++++ client/escalatorclient/v1/shell.py | 56 +++++++ 9 files changed, 449 insertions(+) create mode 100644 api/escalator/api/v1/clusters.py create mode 100644 api/escalator/api/v1/controller.py create mode 100644 api/escalator/installer/__init__.py create mode 100644 api/escalator/installer/daisy/__init__.py create mode 100644 api/escalator/installer/daisy/api.py create mode 100644 client/escalatorclient/v1/clusters.py diff --git a/api/escalator/api/v1/clusters.py b/api/escalator/api/v1/clusters.py new file mode 100644 index 0000000..c37dda1 --- /dev/null +++ b/api/escalator/api/v1/clusters.py @@ -0,0 +1,123 @@ +# Copyright 2013 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +""" +/clusters list for Escalator v1 API +""" +from oslo_log import log as logging +from webob.exc import HTTPBadRequest +from webob.exc import HTTPForbidden + +from escalator.api import policy +from escalator.api.v1 import controller +from escalator.common import exception +from escalator.common import utils +from escalator.common import wsgi +from escalator import i18n +from escalator import notifier +import escalator.installer.daisy.api as daisy_api + +LOG = logging.getLogger(__name__) +_ = i18n._ +_LE = i18n._LE +_LI = i18n._LI +_LW = i18n._LW + + +class Controller(controller.BaseController): + """ + WSGI controller for clusters resource in Escalaotr v1 API + + The clusters resource API is a RESTful web service for cluster data. + The API is as follows:: + + GET /clusters -- Returns a set of brief metadata about clusters + GET /clusters -- Returns a set of detailed metadata about + clusters + """ + def __init__(self): + self.notifier = notifier.Notifier() + self.policy = policy.Enforcer() + + def _enforce(self, req, action, target=None): + """Authorize an action against our policies""" + if target is None: + target = {} + try: + self.policy.enforce(req.context, action, target) + except exception.Forbidden: + raise HTTPForbidden() + + def detail(self, req): + """ + Returns detailed information for all available clusters + + :param req: The WSGI/Webob Request object + :retval The response body is a mapping of the following form:: + + {'clusters': [ + {'id': , + 'name': , + 'nodes': , + 'networks': , + 'description': , + 'created_at': , + 'updated_at': , + 'deleted_at': |,}, ... + ]} + """ + self._enforce(req, 'get_clusters') + try: + clusters = daisy_api.cluster_list(req.context) + clusters_list = list() + while True: + try: + cluster_new = next(clusters) + clusters_list.append(cluster_new) + except StopIteration: + break + except exception.Invalid as e: + raise HTTPBadRequest(explanation=e.msg, request=req) + return dict(clusters=clusters_list) + + +class ProjectDeserializer(wsgi.JSONRequestDeserializer): + """Handles deserialization of specific controller method requests.""" + + def _deserialize(self, request): + result = {} + result["cluster_meta"] = utils.get_cluster_meta(request) + return result + + +class ProjectSerializer(wsgi.JSONResponseSerializer): + """Handles serialization of specific controller method responses.""" + + def __init__(self): + self.notifier = notifier.Notifier() + + def get_cluster(self, response, result): + cluster_meta = result['cluster_meta'] + response.status = 201 + response.headers['Content-Type'] = 'application/json' + response.body = self.to_json(dict(cluster=cluster_meta)) + return response + + +def create_resource(): + """Projects resource factory method""" + deserializer = ProjectDeserializer() + serializer = ProjectSerializer() + return wsgi.Resource(Controller(), deserializer, serializer) diff --git a/api/escalator/api/v1/controller.py b/api/escalator/api/v1/controller.py new file mode 100644 index 0000000..ad0b9d7 --- /dev/null +++ b/api/escalator/api/v1/controller.py @@ -0,0 +1,13 @@ +class BaseController(object): + + def get_cluster_meta_or_404(self, request, cluster_id): + """ + Grabs the cluster metadata for an cluster with a supplied + identifier or raises an HTTPNotFound (404) response + + :param request: The WSGI/Webob Request object + :param cluster_id: The opaque cluster identifier + + :raises HTTPNotFound if cluster does not exist + """ + pass diff --git a/api/escalator/api/v1/router.py b/api/escalator/api/v1/router.py index e1709ca..5942cb1 100644 --- a/api/escalator/api/v1/router.py +++ b/api/escalator/api/v1/router.py @@ -14,6 +14,7 @@ # under the License. from escalator.common import wsgi from escalator.api.v1 import versions +from escalator.api.v1 import clusters class API(wsgi.Router): @@ -24,6 +25,12 @@ class API(wsgi.Router): wsgi.Resource(wsgi.RejectMethodController()) versions_resource = versions.create_resource() + clusters_resource = clusters.create_resource() + + mapper.connect("/clusters", + controller=clusters_resource, + action='detail', + conditions={'method': ['GET']}) mapper.connect("/versions", controller=versions_resource, diff --git a/api/escalator/installer/__init__.py b/api/escalator/installer/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/escalator/installer/daisy/__init__.py b/api/escalator/installer/daisy/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/api/escalator/installer/daisy/api.py b/api/escalator/installer/daisy/api.py new file mode 100644 index 0000000..661d0d5 --- /dev/null +++ b/api/escalator/installer/daisy/api.py @@ -0,0 +1,14 @@ +from daisyclient.v1 import client as daisy_client + + +def daisyclient(request): + DAISY_ENDPOINT_URL = "http://127.0.0.1:19292" + return daisy_client.Client(version=1, endpoint=DAISY_ENDPOINT_URL) + + +def cluster_list(request): + return daisyclient(request).clusters.list() + + +def cluster_get(request, cluster_id): + return daisyclient(request).clusters.get(cluster_id) diff --git a/client/escalatorclient/v1/client.py b/client/escalatorclient/v1/client.py index d5bf6bc..974fc4e 100644 --- a/client/escalatorclient/v1/client.py +++ b/client/escalatorclient/v1/client.py @@ -16,6 +16,7 @@ from escalatorclient.common import http from escalatorclient.common import utils from escalatorclient.v1.versions import VersionManager +from escalatorclient.v1.clusters import ClusterManager class Client(object): @@ -34,3 +35,4 @@ class Client(object): self.version = version or 1.0 self.http_client = http.HTTPClient(endpoint, *args, **kwargs) self.versions = VersionManager(self.http_client) + self.clusters = ClusterManager(self.http_client) diff --git a/client/escalatorclient/v1/clusters.py b/client/escalatorclient/v1/clusters.py new file mode 100644 index 0000000..8877d8b --- /dev/null +++ b/client/escalatorclient/v1/clusters.py @@ -0,0 +1,234 @@ +# Copyright 2012 OpenStack Foundation +# All Rights Reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); you may +# not use this file except in compliance with the License. You may obtain +# a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, WITHOUT +# WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the +# License for the specific language governing permissions and limitations +# under the License. + +import copy + +from oslo_utils import encodeutils +from oslo_utils import strutils +import six +import six.moves.urllib.parse as urlparse + +from escalatorclient.common import utils +from escalatorclient.openstack.common.apiclient import base + +DEFAULT_PAGE_SIZE = 20 + +SORT_DIR_VALUES = ('asc', 'desc') +SORT_KEY_VALUES = ('name', 'auto_scale', 'id', 'created_at', 'updated_at') + +OS_REQ_ID_HDR = 'x-openstack-request-id' + + +class Cluster(base.Resource): + + def __repr__(self): + return "" % self._info + + def data(self, **kwargs): + return self.manager.data(self, **kwargs) + + +class ClusterManager(base.ManagerWithFind): + resource_class = Cluster + + def _list(self, url, response_key, obj_class=None, body=None): + resp, body = self.client.get(url) + + if obj_class is None: + obj_class = self.resource_class + + data = body[response_key] + return ([obj_class(self, res, loaded=True) for res in data if res], + resp) + + def _cluster_meta_from_headers(self, headers): + meta = {'properties': {}} + safe_decode = encodeutils.safe_decode + for key, value in six.iteritems(headers): + value = safe_decode(value, incoming='utf-8') + if key.startswith('x-image-meta-property-'): + _key = safe_decode(key[22:], incoming='utf-8') + meta['properties'][_key] = value + elif key.startswith('x-image-meta-'): + _key = safe_decode(key[13:], incoming='utf-8') + meta[_key] = value + + for key in ['is_public', 'protected', 'deleted']: + if key in meta: + meta[key] = strutils.bool_from_string(meta[key]) + + return self._format_cluster_meta_for_user(meta) + + def _cluster_meta_to_headers(self, fields): + headers = {} + fields_copy = copy.deepcopy(fields) + + # NOTE(flaper87): Convert to str, headers + # that are not instance of basestring. All + # headers will be encoded later, before the + # request is sent. + + for key, value in six.iteritems(fields_copy): + headers['%s' % key] = utils.to_str(value) + return headers + + @staticmethod + def _format_cluster_meta_for_user(meta): + for key in ['size', 'min_ram', 'min_disk']: + if key in meta: + try: + meta[key] = int(meta[key]) if meta[key] else 0 + except ValueError: + pass + return meta + + def get(self, cluster, **kwargs): + """Get the metadata for a specific cluster. + + :param cluster: host object or id to look up + :rtype: :class:`Cluster` + """ + cluster_id = base.getid(cluster) + resp, body = self.client.get('/v1/clusters/%s' + % urlparse.quote(str(cluster_id))) + # meta = self._cluster_meta_from_headers(resp.headers) + return_request_id = kwargs.get('return_req_id', None) + if return_request_id is not None: + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) + # return Host(self, meta) + return Cluster(self, self._format_cluster_meta_for_user( + body['cluster'])) + + def data(self, image, do_checksum=True, **kwargs): + """Get the raw data for a specific image. + + :param image: image object or id to look up + :param do_checksum: Enable/disable checksum validation + :rtype: iterable containing image data + """ + image_id = base.getid(image) + resp, body = self.client.get('/v1/images/%s' + % urlparse.quote(str(image_id))) + content_length = int(resp.headers.get('content-length', 0)) + checksum = resp.headers.get('x-image-meta-checksum', None) + if do_checksum and checksum is not None: + body = utils.integrity_iter(body, checksum) + return_request_id = kwargs.get('return_req_id', None) + if return_request_id is not None: + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) + + return utils.IterableWithLength(body, content_length) + + def _build_params(self, parameters): + params = {'limit': parameters.get('page_size', DEFAULT_PAGE_SIZE)} + + if 'marker' in parameters: + params['marker'] = parameters['marker'] + + sort_key = parameters.get('sort_key') + if sort_key is not None: + if sort_key in SORT_KEY_VALUES: + params['sort_key'] = sort_key + else: + raise ValueError('sort_key must be one of the following: %s.' + % ', '.join(SORT_KEY_VALUES)) + + sort_dir = parameters.get('sort_dir') + if sort_dir is not None: + if sort_dir in SORT_DIR_VALUES: + params['sort_dir'] = sort_dir + else: + raise ValueError('sort_dir must be one of the following: %s.' + % ', '.join(SORT_DIR_VALUES)) + + filters = parameters.get('filters', {}) + params.update(filters) + + return params + + def list(self, **kwargs): + """Get a list of clusters. + + :param page_size: number of items to request in each paginated request + :param limit: maximum number of clusters to return + :param marker: begin returning clusters that + appear later in the cluster + list than that represented by this cluster id + :param filters: dict of direct comparison filters that mimics the + structure of an cluster object + :param return_request_id: If an empty list is provided, populate this + list with the request ID value from the header + x-openstack-request-id + :rtype: list of :class:`Cluster` + """ + absolute_limit = kwargs.get('limit') + page_size = kwargs.get('page_size', DEFAULT_PAGE_SIZE) + + def paginate(qp, return_request_id=None): + for param, value in six.iteritems(qp): + if isinstance(value, six.string_types): + # Note(flaper87) Url encoding should + # be moved inside http utils, at least + # shouldn't be here. + # + # Making sure all params are str before + # trying to encode them + qp[param] = encodeutils.safe_decode(value) + + url = '/v1/clusters?%s' % urlparse.urlencode(qp) + clusters, resp = self._list(url, "clusters") + + if return_request_id is not None: + return_request_id.append(resp.headers.get(OS_REQ_ID_HDR, None)) + + for cluster in clusters: + yield cluster + + return_request_id = kwargs.get('return_req_id', None) + + params = self._build_params(kwargs) + + seen = 0 + while True: + seen_last_page = 0 + filtered = 0 + for cluster in paginate(params, return_request_id): + last_cluster = cluster.id + + if (absolute_limit is not None and + seen + seen_last_page >= absolute_limit): + # Note(kragniz): we've seen enough images + return + else: + seen_last_page += 1 + yield cluster + + seen += seen_last_page + + if seen_last_page + filtered == 0: + # Note(kragniz): we didn't get any clusters in the last page + return + + if absolute_limit is not None and seen >= absolute_limit: + # Note(kragniz): reached the limit of clusters to return + return + + if page_size and seen_last_page + filtered < page_size: + # Note(kragniz): we've reached the last page of the clusters + return + + # Note(kragniz): there are more clusters to come + params['marker'] = last_cluster + seen_last_page = 0 diff --git a/client/escalatorclient/v1/shell.py b/client/escalatorclient/v1/shell.py index 401ad76..10aa1bc 100644 --- a/client/escalatorclient/v1/shell.py +++ b/client/escalatorclient/v1/shell.py @@ -19,6 +19,7 @@ import copy import functools from oslo_utils import strutils import escalatorclient.v1.versions +import escalatorclient.v1.clusters from escalatorclient.common import utils _bool_strict = functools.partial(strutils.bool_from_string, strict=True) @@ -82,3 +83,58 @@ def do_cluster_version_list(dc, args): 'checksum', 'description', 'status', 'VERSION_PATCH'] utils.print_list(versions, columns) + + +@utils.arg('--name', metavar='', + help='Filter version to those that have this name.') +@utils.arg('--status', metavar='', + help='Filter version status.') +@utils.arg('--type', metavar='', + help='Filter by type.') +@utils.arg('--version', metavar='', + help='Filter by version number.') +@utils.arg('--page-size', metavar='', default=None, type=int, + help='Number to request in each paginated request.') +@utils.arg('--sort-key', default='name', + choices=escalatorclient.v1.versions.SORT_KEY_VALUES, + help='Sort version list by specified field.') +@utils.arg('--sort-dir', default='asc', + choices=escalatorclient.v1.versions.SORT_DIR_VALUES, + help='Sort version list in specified direction.') +def do_cluster_list(gc, args): + """List clusters you can access.""" + filter_keys = ['name'] + filter_items = [(key, getattr(args, key)) for key in filter_keys] + filters = dict([item for item in filter_items if item[1] is not None]) + + kwargs = {'filters': filters} + if args.page_size is not None: + kwargs['page_size'] = args.page_size + + kwargs['sort_key'] = args.sort_key + kwargs['sort_dir'] = args.sort_dir + + clusters = gc.clusters.list(**kwargs) + + columns = ['ID', 'Name', 'Description', 'Nodes', 'Networks', + 'Auto_scale', 'Use_dns', 'Status'] + utils.print_list(clusters, columns) + + +@utils.arg('id', metavar='', + help='Filter cluster to those that have this id.') +def do_cluster_detail(gc, args): + """List cluster you can access.""" + filter_keys = ['id'] + filter_items = [(key, getattr(args, key)) for key in filter_keys] + filters = dict([item for item in filter_items if item[1] is not None]) + fields = dict(filter(lambda x: x[1] is not None, vars(args).items())) + kwargs = {'filters': filters} + if filters: + cluster = utils.find_resource(gc.clusters, fields.pop('id')) + _escalator_show(cluster) + else: + cluster = gc.clusters.list(**kwargs) + columns = ['ID', 'Name', 'Description', 'Nodes', + 'Networks', 'Auto_scale', 'Use_dns'] + utils.print_list(cluster, columns) -- 2.16.6