2 # SPDX-license-identifier: Apache-2.0
3 ##############################################################################
4 # Copyright (c) 2016 The Linux Foundation and others
6 # Licensed under the Apache License, Version 2.0 (the "License");
7 # you may not use this file except in compliance with the License.
8 # You may obtain a copy of the License at
9 # http://www.apache.org/licenses/LICENSE-2.0
11 # Unless required by applicable law or agreed to in writing, software
12 # distributed under the License is distributed on an "AS IS" BASIS,
13 # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
14 # See the License for the specific language governing permissions and
15 # limitations under the License.
16 ##############################################################################
19 Generate JSON listing of OPNFV Artifacts
21 This produces a slimmed down version of metadata provided by Google
22 Storage for each artifact. Also excludes a large number of uninteresting
26 from apiclient import discovery
27 from apiclient.errors import HttpError
45 # List of file extensions to filter out
64 parser = argparse.ArgumentParser(
65 description='OPNFV Artifacts JSON Generator')
71 help='API Key for Google Cloud Storage')
79 help='pretty print the output')
81 # Parse and assign arguments
82 args = parser.parse_args()
84 pretty_print = args.pretty
87 def output(item, indent=2):
88 print(json.dumps(item, sort_keys=True, indent=indent))
91 def has_gerrit_review(dir_list):
93 If a directory contains an integer, it is assumed to be a gerrit
102 def has_release(dir_list):
104 Checks if any directory contains a release name
112 def has_documentation(dir_list):
114 Checks for a directory specifically named 'docs'
122 # Rename this or modify how gerrit review are handled
123 def has_logs(gerrit_review):
125 If a gerrit review exists, create a link to the review
128 return "https://gerrit.opnfv.org/gerrit/#/c/%s" % gerrit_review
132 def has_ignorable_extension(filename):
133 for extension in ignore_extensions:
134 if filename.lower().endswith(extension):
139 def get_results(key):
141 Pull down all metadata from artifacts.opnfv.org
142 and store it in projects as:
143 { 'PROJECT': [file ...], }
145 storage = discovery.build('storage', 'v1', developerKey=key)
146 files = storage.objects().list(bucket='artifacts.opnfv.org',
147 fields='nextPageToken,'
155 while (files is not None):
156 sites = files.execute()
158 for site in sites['items']:
159 # Filter out unneeded files (js, images, css, buildinfo, etc)
160 if has_ignorable_extension(site['name']):
163 # Split /foo/bar/ into ['foo', 'bar'] and remove any extra
164 # slashes (ex. /foo//bar/)
165 site_split = filter(None, site['name'].split('/'))
167 # Don't do anything if we aren't given files multiple
169 if len(site_split) < 2:
172 project = site_split[0]
173 name = '/'.join(site_split[1:])
174 proxy = "http://build.opnfv.org/artifacts.opnfv.org/%s" % site[
176 if name.endswith('.html'):
177 href = "http://artifacts.opnfv.org/%s" % site['name']
180 href = site['mediaLink']
181 href_type = 'download'
183 gerrit = has_gerrit_review(site_split)
184 logs = False # has_logs(gerrit)
185 # documentation = has_documentation(site_split)
186 release = has_release(site_split)
197 'category': category,
198 'gerritreview': gerrit,
201 'size': site['size'],
202 'time': site['updated'],
203 'contentType': site['contentType'],
205 'href_type': href_type,
209 if project in releases:
210 if project not in api['releases']:
211 api['releases'][project] = [metadata]
213 api['releases'][project].append(metadata)
215 if project not in api['projects']:
216 api['projects'][project] = [metadata]
218 api['projects'][project].append(metadata)
220 files = storage.objects().list_next(files, sites)
225 # Fail if there is an invalid response from GCE
227 js = get_results(key)
228 except HttpError as e:
229 print >> sys.stderr, e
232 output(js, indent=pretty_print)