1 // -*- mode:C++; tab-width:8; c-basic-offset:2; indent-tabs-mode:t -*-
2 // vim: ts=8 sw=2 smarttab
9 #include <boost/optional.hpp>
11 #include "auth/Crypto.h"
12 #include "compressor/Compressor.h"
14 #include "common/armor.h"
15 #include "common/ceph_json.h"
16 #include "common/config.h"
17 #include "common/ceph_argparse.h"
18 #include "common/Formatter.h"
19 #include "common/errno.h"
20 #include "common/safe_io.h"
22 #include "cls/rgw/cls_rgw_client.h"
24 #include "global/global_init.h"
26 #include "include/utime.h"
27 #include "include/str_list.h"
30 #include "rgw_bucket.h"
31 #include "rgw_rados.h"
33 #include "rgw_acl_s3.h"
36 #include "rgw_formats.h"
37 #include "rgw_usage.h"
38 #include "rgw_replica_log.h"
39 #include "rgw_orphan.h"
41 #include "rgw_data_sync.h"
42 #include "rgw_rest_conn.h"
43 #include "rgw_realm_watcher.h"
45 #include "rgw_reshard.h"
49 #define dout_context g_ceph_context
50 #define dout_subsys ceph_subsys_rgw
52 #define SECRET_KEY_LEN 40
53 #define PUBLIC_ID_LEN 20
55 static RGWRados *store = NULL;
59 cout << "usage: radosgw-admin <cmd> [options...]" << std::endl;
60 cout << "commands:\n";
61 cout << " user create create a new user\n" ;
62 cout << " user modify modify user\n";
63 cout << " user info get user info\n";
64 cout << " user rm remove user\n";
65 cout << " user suspend suspend a user\n";
66 cout << " user enable re-enable user after suspension\n";
67 cout << " user check check user info\n";
68 cout << " user stats show user stats as accounted by quota subsystem\n";
69 cout << " user list list users\n";
70 cout << " caps add add user capabilities\n";
71 cout << " caps rm remove user capabilities\n";
72 cout << " subuser create create a new subuser\n" ;
73 cout << " subuser modify modify subuser\n";
74 cout << " subuser rm remove subuser\n";
75 cout << " key create create access key\n";
76 cout << " key rm remove access key\n";
77 cout << " bucket list list buckets\n";
78 cout << " bucket limit check show bucket sharding stats\n";
79 cout << " bucket link link bucket to specified user\n";
80 cout << " bucket unlink unlink bucket from specified user\n";
81 cout << " bucket stats returns bucket statistics\n";
82 cout << " bucket rm remove bucket\n";
83 cout << " bucket check check bucket index\n";
84 cout << " bucket reshard reshard bucket\n";
85 cout << " bucket sync disable disable bucket sync\n";
86 cout << " bucket sync enable enable bucket sync\n";
87 cout << " bi get retrieve bucket index object entries\n";
88 cout << " bi put store bucket index object entries\n";
89 cout << " bi list list raw bucket index entries\n";
90 cout << " object rm remove object\n";
91 cout << " object stat stat an object for its metadata\n";
92 cout << " object unlink unlink object from bucket index\n";
93 cout << " objects expire run expired objects cleanup\n";
94 cout << " period delete delete a period\n";
95 cout << " period get get period info\n";
96 cout << " period get-current get current period info\n";
97 cout << " period pull pull a period\n";
98 cout << " period push push a period\n";
99 cout << " period list list all periods\n";
100 cout << " period update update the staging period\n";
101 cout << " period commit commit the staging period\n";
102 cout << " quota set set quota params\n";
103 cout << " quota enable enable quota\n";
104 cout << " quota disable disable quota\n";
105 cout << " global quota get view global quota params\n";
106 cout << " global quota set set global quota params\n";
107 cout << " global quota enable enable a global quota\n";
108 cout << " global quota disable disable a global quota\n";
109 cout << " realm create create a new realm\n";
110 cout << " realm delete delete a realm\n";
111 cout << " realm get show realm info\n";
112 cout << " realm get-default get default realm name\n";
113 cout << " realm list list realms\n";
114 cout << " realm list-periods list all realm periods\n";
115 cout << " realm remove remove a zonegroup from the realm\n";
116 cout << " realm rename rename a realm\n";
117 cout << " realm set set realm info (requires infile)\n";
118 cout << " realm default set realm as default\n";
119 cout << " realm pull pull a realm and its current period\n";
120 cout << " zonegroup add add a zone to a zonegroup\n";
121 cout << " zonegroup create create a new zone group info\n";
122 cout << " zonegroup default set default zone group\n";
123 cout << " zonegroup delete delete a zone group info\n";
124 cout << " zonegroup get show zone group info\n";
125 cout << " zonegroup modify modify an existing zonegroup\n";
126 cout << " zonegroup set set zone group info (requires infile)\n";
127 cout << " zonegroup remove remove a zone from a zonegroup\n";
128 cout << " zonegroup rename rename a zone group\n";
129 cout << " zonegroup list list all zone groups set on this cluster\n";
130 cout << " zonegroup placement list list zonegroup's placement targets\n";
131 cout << " zonegroup placement add add a placement target id to a zonegroup\n";
132 cout << " zonegroup placement modify modify a placement target of a specific zonegroup\n";
133 cout << " zonegroup placement rm remove a placement target from a zonegroup\n";
134 cout << " zonegroup placement default set a zonegroup's default placement target\n";
135 cout << " zone create create a new zone\n";
136 cout << " zone delete delete a zone\n";
137 cout << " zone get show zone cluster params\n";
138 cout << " zone modify modify an existing zone\n";
139 cout << " zone set set zone cluster params (requires infile)\n";
140 cout << " zone list list all zones set on this cluster\n";
141 cout << " zone rename rename a zone\n";
142 cout << " zone placement list list zone's placement targets\n";
143 cout << " zone placement add add a zone placement target\n";
144 cout << " zone placement modify modify a zone placement target\n";
145 cout << " zone placement rm remove a zone placement target\n";
146 cout << " pool add add an existing pool for data placement\n";
147 cout << " pool rm remove an existing pool from data placement set\n";
148 cout << " pools list list placement active set\n";
149 cout << " policy read bucket/object policy\n";
150 cout << " log list list log objects\n";
151 cout << " log show dump a log from specific object or (bucket + date\n";
152 cout << " + bucket-id)\n";
153 cout << " (NOTE: required to specify formatting of date\n";
154 cout << " to \"YYYY-MM-DD-hh\")\n";
155 cout << " log rm remove log object\n";
156 cout << " usage show show usage (by user, date range)\n";
157 cout << " usage trim trim usage (by user, date range)\n";
158 cout << " gc list dump expired garbage collection objects (specify\n";
159 cout << " --include-all to list all entries, including unexpired)\n";
160 cout << " gc process manually process garbage\n";
161 cout << " lc list list all bucket lifecycle progress\n";
162 cout << " lc process manually process lifecycle\n";
163 cout << " metadata get get metadata info\n";
164 cout << " metadata put put metadata info\n";
165 cout << " metadata rm remove metadata info\n";
166 cout << " metadata list list metadata info\n";
167 cout << " mdlog list list metadata log\n";
168 cout << " mdlog trim trim metadata log (use start-date, end-date or\n";
169 cout << " start-marker, end-marker)\n";
170 cout << " mdlog status read metadata log status\n";
171 cout << " bilog list list bucket index log\n";
172 cout << " bilog trim trim bucket index log (use start-marker, end-marker)\n";
173 cout << " datalog list list data log\n";
174 cout << " datalog trim trim data log\n";
175 cout << " datalog status read data log status\n";
176 cout << " opstate list list stateful operations entries (use client_id,\n";
177 cout << " op_id, object)\n";
178 cout << " opstate set set state on an entry (use client_id, op_id, object, state)\n";
179 cout << " opstate renew renew state on an entry (use client_id, op_id, object)\n";
180 cout << " opstate rm remove entry (use client_id, op_id, object)\n";
181 cout << " replicalog get get replica metadata log entry\n";
182 cout << " replicalog update update replica metadata log entry\n";
183 cout << " replicalog delete delete replica metadata log entry\n";
184 cout << " orphans find init and run search for leaked rados objects (use job-id, pool)\n";
185 cout << " orphans finish clean up search for leaked rados objects\n";
186 cout << " orphans list-jobs list the current job-ids for orphans search\n";
187 cout << " role create create a AWS role for use with STS\n";
188 cout << " role delete delete a role\n";
189 cout << " role get get a role\n";
190 cout << " role list list roles with specified path prefix\n";
191 cout << " role modify modify the assume role policy of an existing role\n";
192 cout << " role-policy put add/update permission policy to role\n";
193 cout << " role-policy list list policies attached to a role\n";
194 cout << " role-policy get get the specified inline policy document embedded with the given role\n";
195 cout << " role-policy delete delete policy attached to a role\n";
196 cout << " reshard add schedule a resharding of a bucket\n";
197 cout << " reshard list list all bucket resharding or scheduled to be reshared\n";
198 cout << " reshard process process of scheduled reshard jobs\n";
199 cout << " reshard cancel cancel resharding a bucket\n";
200 cout << "options:\n";
201 cout << " --tenant=<tenant> tenant name\n";
202 cout << " --uid=<id> user id\n";
203 cout << " --subuser=<name> subuser name\n";
204 cout << " --access-key=<key> S3 access key\n";
205 cout << " --email=<email>\n";
206 cout << " --secret/--secret-key=<key>\n";
207 cout << " specify secret key\n";
208 cout << " --gen-access-key generate random access key (for S3)\n";
209 cout << " --gen-secret generate random secret key\n";
210 cout << " --key-type=<type> key type, options are: swift, s3\n";
211 cout << " --temp-url-key[-2]=<key> temp url key\n";
212 cout << " --access=<access> Set access permissions for sub-user, should be one\n";
213 cout << " of read, write, readwrite, full\n";
214 cout << " --display-name=<name>\n";
215 cout << " --max-buckets max number of buckets for a user\n";
216 cout << " --admin set the admin flag on the user\n";
217 cout << " --system set the system flag on the user\n";
218 cout << " --bucket=<bucket>\n";
219 cout << " --pool=<pool>\n";
220 cout << " --object=<object>\n";
221 cout << " --date=<date>\n";
222 cout << " --start-date=<date>\n";
223 cout << " --end-date=<date>\n";
224 cout << " --bucket-id=<bucket-id>\n";
225 cout << " --shard-id=<shard-id> optional for mdlog list\n";
226 cout << " required for: \n";
227 cout << " mdlog trim\n";
228 cout << " replica mdlog get/delete\n";
229 cout << " replica datalog get/delete\n";
230 cout << " --metadata-key=<key> key to retrieve metadata from with metadata get\n";
231 cout << " --remote=<remote> zone or zonegroup id of remote gateway\n";
232 cout << " --period=<id> period id\n";
233 cout << " --epoch=<number> period epoch\n";
234 cout << " --commit commit the period during 'period update'\n";
235 cout << " --staging get staging period info\n";
236 cout << " --master set as master\n";
237 cout << " --master-url master url\n";
238 cout << " --master-zonegroup=<id> master zonegroup id\n";
239 cout << " --master-zone=<id> master zone id\n";
240 cout << " --rgw-realm=<name> realm name\n";
241 cout << " --realm-id=<id> realm id\n";
242 cout << " --realm-new-name=<name> realm new name\n";
243 cout << " --rgw-zonegroup=<name> zonegroup name\n";
244 cout << " --zonegroup-id=<id> zonegroup id\n";
245 cout << " --zonegroup-new-name=<name>\n";
246 cout << " zonegroup new name\n";
247 cout << " --rgw-zone=<name> name of zone in which radosgw is running\n";
248 cout << " --zone-id=<id> zone id\n";
249 cout << " --zone-new-name=<name> zone new name\n";
250 cout << " --source-zone specify the source zone (for data sync)\n";
251 cout << " --default set entity (realm, zonegroup, zone) as default\n";
252 cout << " --read-only set zone as read-only (when adding to zonegroup)\n";
253 cout << " --placement-id placement id for zonegroup placement commands\n";
254 cout << " --tags=<list> list of tags for zonegroup placement add and modify commands\n";
255 cout << " --tags-add=<list> list of tags to add for zonegroup placement modify command\n";
256 cout << " --tags-rm=<list> list of tags to remove for zonegroup placement modify command\n";
257 cout << " --endpoints=<list> zone endpoints\n";
258 cout << " --index-pool=<pool> placement target index pool\n";
259 cout << " --data-pool=<pool> placement target data pool\n";
260 cout << " --data-extra-pool=<pool> placement target data extra (non-ec) pool\n";
261 cout << " --placement-index-type=<type>\n";
262 cout << " placement target index type (normal, indexless, or #id)\n";
263 cout << " --compression=<type> placement target compression type (plugin name or empty/none)\n";
264 cout << " --tier-type=<type> zone tier type\n";
265 cout << " --tier-config=<k>=<v>[,...]\n";
266 cout << " set zone tier config keys, values\n";
267 cout << " --tier-config-rm=<k>[,...]\n";
268 cout << " unset zone tier config keys\n";
269 cout << " --sync-from-all[=false] set/reset whether zone syncs from all zonegroup peers\n";
270 cout << " --sync-from=[zone-name][,...]\n";
271 cout << " set list of zones to sync from\n";
272 cout << " --sync-from-rm=[zone-name][,...]\n";
273 cout << " remove zones from list of zones to sync from\n";
274 cout << " --fix besides checking bucket index, will also fix it\n";
275 cout << " --check-objects bucket check: rebuilds bucket index according to\n";
276 cout << " actual objects state\n";
277 cout << " --format=<format> specify output format for certain operations: xml,\n";
279 cout << " --purge-data when specified, user removal will also purge all the\n";
280 cout << " user data\n";
281 cout << " --purge-keys when specified, subuser removal will also purge all the\n";
282 cout << " subuser keys\n";
283 cout << " --purge-objects remove a bucket's objects before deleting it\n";
284 cout << " (NOTE: required to delete a non-empty bucket)\n";
285 cout << " --sync-stats option to 'user stats', update user stats with current\n";
286 cout << " stats reported by user's buckets indexes\n";
287 cout << " --show-log-entries=<flag> enable/disable dump of log entries on log show\n";
288 cout << " --show-log-sum=<flag> enable/disable dump of log summation on log show\n";
289 cout << " --skip-zero-entries log show only dumps entries that don't have zero value\n";
290 cout << " in one of the numeric field\n";
291 cout << " --infile=<file> specify a file to read in when setting data\n";
292 cout << " --state=<state string> specify a state for the opstate set command\n";
293 cout << " --replica-log-type replica log type (metadata, data, bucket), required for\n";
294 cout << " replica log operations\n";
295 cout << " --categories=<list> comma separated list of categories, used in usage show\n";
296 cout << " --caps=<caps> list of caps (e.g., \"usage=read, write; user=read\")\n";
297 cout << " --yes-i-really-mean-it required for certain operations\n";
298 cout << " --warnings-only when specified with bucket limit check, list\n";
299 cout << " only buckets nearing or over the current max\n";
300 cout << " objects per shard value\n";
301 cout << " --bypass-gc when specified with bucket deletion, triggers\n";
302 cout << " object deletions by not involving GC\n";
303 cout << " --inconsistent-index when specified with bucket deletion and bypass-gc set to true,\n";
304 cout << " ignores bucket index consistency\n";
306 cout << "<date> := \"YYYY-MM-DD[ hh:mm:ss]\"\n";
307 cout << "\nQuota options:\n";
308 cout << " --bucket specified bucket for quota command\n";
309 cout << " --max-objects specify max objects (negative value to disable)\n";
310 cout << " --max-size specify max size (in B/K/M/G/T, negative value to disable)\n";
311 cout << " --quota-scope scope of quota (bucket, user)\n";
312 cout << "\nOrphans search options:\n";
313 cout << " --pool data pool to scan for leaked rados objects in\n";
314 cout << " --num-shards num of shards to use for keeping the temporary scan info\n";
315 cout << " --orphan-stale-secs num of seconds to wait before declaring an object to be an orphan (default: 86400)\n";
316 cout << " --job-id set the job id (for orphans find)\n";
317 cout << " --max-concurrent-ios maximum concurrent ios for orphans find (default: 32)\n";
318 cout << "\nOrphans list-jobs options:\n";
319 cout << " --extra-info provide extra info in job list\n";
320 cout << "\nRole options:\n";
321 cout << " --role-name name of the role to create\n";
322 cout << " --path path to the role\n";
323 cout << " --assume-role-policy-doc the trust relationship policy document that grants an entity permission to assume the role\n";
324 cout << " --policy-name name of the policy document\n";
325 cout << " --policy-doc permission policy document\n";
326 cout << " --path-prefix path prefix for filtering roles\n";
328 generic_client_usage();
348 OPT_BUCKET_LIMIT_CHECK,
353 OPT_BUCKET_SYNC_STATUS,
354 OPT_BUCKET_SYNC_INIT,
356 OPT_BUCKET_SYNC_DISABLE,
357 OPT_BUCKET_SYNC_ENABLE,
390 OPT_ORPHANS_LIST_JOBS,
392 OPT_ZONEGROUP_CREATE,
393 OPT_ZONEGROUP_DEFAULT,
394 OPT_ZONEGROUP_DELETE,
396 OPT_ZONEGROUP_MODIFY,
399 OPT_ZONEGROUP_REMOVE,
400 OPT_ZONEGROUP_RENAME,
401 OPT_ZONEGROUP_PLACEMENT_ADD,
402 OPT_ZONEGROUP_PLACEMENT_MODIFY,
403 OPT_ZONEGROUP_PLACEMENT_RM,
404 OPT_ZONEGROUP_PLACEMENT_LIST,
405 OPT_ZONEGROUP_PLACEMENT_DEFAULT,
414 OPT_ZONE_PLACEMENT_ADD,
415 OPT_ZONE_PLACEMENT_MODIFY,
416 OPT_ZONE_PLACEMENT_RM,
417 OPT_ZONE_PLACEMENT_LIST,
424 OPT_METADATA_SYNC_STATUS,
425 OPT_METADATA_SYNC_INIT,
426 OPT_METADATA_SYNC_RUN,
436 OPT_DATA_SYNC_STATUS,
447 OPT_REPLICALOG_UPDATE,
448 OPT_REPLICALOG_DELETE,
452 OPT_REALM_GET_DEFAULT,
454 OPT_REALM_LIST_PERIODS,
462 OPT_PERIOD_GET_CURRENT,
468 OPT_GLOBAL_QUOTA_GET,
469 OPT_GLOBAL_QUOTA_SET,
470 OPT_GLOBAL_QUOTA_ENABLE,
471 OPT_GLOBAL_QUOTA_DISABLE,
479 OPT_ROLE_POLICY_LIST,
481 OPT_ROLE_POLICY_DELETE,
489 static int get_cmd(const char *cmd, const char *prev_cmd, const char *prev_prev_cmd, bool *need_more)
492 // NOTE: please keep the checks in alphabetical order !!!
493 if (strcmp(cmd, "bi") == 0 ||
494 strcmp(cmd, "bilog") == 0 ||
495 strcmp(cmd, "buckets") == 0 ||
496 strcmp(cmd, "caps") == 0 ||
497 strcmp(cmd, "data") == 0 ||
498 strcmp(cmd, "datalog") == 0 ||
499 strcmp(cmd, "error") == 0 ||
500 strcmp(cmd, "gc") == 0 ||
501 strcmp(cmd, "global") == 0 ||
502 strcmp(cmd, "key") == 0 ||
503 strcmp(cmd, "log") == 0 ||
504 strcmp(cmd, "lc") == 0 ||
505 strcmp(cmd, "mdlog") == 0 ||
506 strcmp(cmd, "metadata") == 0 ||
507 strcmp(cmd, "object") == 0 ||
508 strcmp(cmd, "objects") == 0 ||
509 strcmp(cmd, "olh") == 0 ||
510 strcmp(cmd, "opstate") == 0 ||
511 strcmp(cmd, "orphans") == 0 ||
512 strcmp(cmd, "period") == 0 ||
513 strcmp(cmd, "placement") == 0 ||
514 strcmp(cmd, "pool") == 0 ||
515 strcmp(cmd, "pools") == 0 ||
516 strcmp(cmd, "quota") == 0 ||
517 strcmp(cmd, "realm") == 0 ||
518 strcmp(cmd, "replicalog") == 0 ||
519 strcmp(cmd, "role") == 0 ||
520 strcmp(cmd, "role-policy") == 0 ||
521 strcmp(cmd, "subuser") == 0 ||
522 strcmp(cmd, "sync") == 0 ||
523 strcmp(cmd, "usage") == 0 ||
524 strcmp(cmd, "user") == 0 ||
525 strcmp(cmd, "zone") == 0 ||
526 strcmp(cmd, "zonegroup") == 0 ||
527 strcmp(cmd, "zonegroups") == 0) {
533 * can do both radosgw-admin bucket reshard, and radosgw-admin reshard bucket
535 if (strcmp(cmd, "reshard") == 0 &&
536 !(prev_cmd && strcmp(prev_cmd, "bucket") == 0)) {
540 if (strcmp(cmd, "bucket") == 0 &&
541 !(prev_cmd && strcmp(prev_cmd, "reshard") == 0)) {
546 if (strcmp(cmd, "policy") == 0)
552 if (strcmp(prev_cmd, "user") == 0) {
553 if (strcmp(cmd, "create") == 0)
554 return OPT_USER_CREATE;
555 if (strcmp(cmd, "info") == 0)
556 return OPT_USER_INFO;
557 if (strcmp(cmd, "modify") == 0)
558 return OPT_USER_MODIFY;
559 if (strcmp(cmd, "rm") == 0)
561 if (strcmp(cmd, "suspend") == 0)
562 return OPT_USER_SUSPEND;
563 if (strcmp(cmd, "enable") == 0)
564 return OPT_USER_ENABLE;
565 if (strcmp(cmd, "check") == 0)
566 return OPT_USER_CHECK;
567 if (strcmp(cmd, "stats") == 0)
568 return OPT_USER_STATS;
569 if (strcmp(cmd, "list") == 0)
570 return OPT_USER_LIST;
571 } else if (strcmp(prev_cmd, "subuser") == 0) {
572 if (strcmp(cmd, "create") == 0)
573 return OPT_SUBUSER_CREATE;
574 if (strcmp(cmd, "modify") == 0)
575 return OPT_SUBUSER_MODIFY;
576 if (strcmp(cmd, "rm") == 0)
577 return OPT_SUBUSER_RM;
578 } else if (strcmp(prev_cmd, "key") == 0) {
579 if (strcmp(cmd, "create") == 0)
580 return OPT_KEY_CREATE;
581 if (strcmp(cmd, "rm") == 0)
583 } else if (strcmp(prev_cmd, "buckets") == 0) {
584 if (strcmp(cmd, "list") == 0)
585 return OPT_BUCKETS_LIST;
586 } else if (strcmp(prev_cmd, "bucket") == 0) {
587 if (strcmp(cmd, "list") == 0)
588 return OPT_BUCKETS_LIST;
589 if (strcmp(cmd, "link") == 0)
590 return OPT_BUCKET_LINK;
591 if (strcmp(cmd, "unlink") == 0)
592 return OPT_BUCKET_UNLINK;
593 if (strcmp(cmd, "stats") == 0)
594 return OPT_BUCKET_STATS;
595 if (strcmp(cmd, "rm") == 0)
596 return OPT_BUCKET_RM;
597 if (strcmp(cmd, "rewrite") == 0)
598 return OPT_BUCKET_REWRITE;
599 if (strcmp(cmd, "reshard") == 0)
600 return OPT_BUCKET_RESHARD;
601 if (strcmp(cmd, "check") == 0)
602 return OPT_BUCKET_CHECK;
603 if (strcmp(cmd, "sync") == 0) {
607 if (strcmp(cmd, "limit") == 0) {
611 } else if (prev_prev_cmd && strcmp(prev_prev_cmd, "bucket") == 0) {
612 if (strcmp(prev_cmd, "sync") == 0) {
613 if (strcmp(cmd, "status") == 0)
614 return OPT_BUCKET_SYNC_STATUS;
615 if (strcmp(cmd, "init") == 0)
616 return OPT_BUCKET_SYNC_INIT;
617 if (strcmp(cmd, "run") == 0)
618 return OPT_BUCKET_SYNC_RUN;
619 if (strcmp(cmd, "disable") == 0)
620 return OPT_BUCKET_SYNC_DISABLE;
621 if (strcmp(cmd, "enable") == 0)
622 return OPT_BUCKET_SYNC_ENABLE;
623 } else if ((strcmp(prev_cmd, "limit") == 0) &&
624 (strcmp(cmd, "check") == 0)) {
625 return OPT_BUCKET_LIMIT_CHECK;
627 } else if (strcmp(prev_cmd, "log") == 0) {
628 if (strcmp(cmd, "list") == 0)
630 if (strcmp(cmd, "show") == 0)
632 if (strcmp(cmd, "rm") == 0)
634 } else if (strcmp(prev_cmd, "usage") == 0) {
635 if (strcmp(cmd, "show") == 0)
636 return OPT_USAGE_SHOW;
637 if (strcmp(cmd, "trim") == 0)
638 return OPT_USAGE_TRIM;
639 } else if (strcmp(prev_cmd, "caps") == 0) {
640 if (strcmp(cmd, "add") == 0)
642 if (strcmp(cmd, "rm") == 0)
644 } else if (strcmp(prev_cmd, "pool") == 0) {
645 if (strcmp(cmd, "add") == 0)
647 if (strcmp(cmd, "rm") == 0)
649 if (strcmp(cmd, "list") == 0)
650 return OPT_POOLS_LIST;
651 } else if (strcmp(prev_cmd, "pools") == 0) {
652 if (strcmp(cmd, "list") == 0)
653 return OPT_POOLS_LIST;
654 } else if (strcmp(prev_cmd, "object") == 0) {
655 if (strcmp(cmd, "rm") == 0)
656 return OPT_OBJECT_RM;
657 if (strcmp(cmd, "unlink") == 0)
658 return OPT_OBJECT_UNLINK;
659 if (strcmp(cmd, "stat") == 0)
660 return OPT_OBJECT_STAT;
661 if (strcmp(cmd, "rewrite") == 0)
662 return OPT_OBJECT_REWRITE;
663 } else if (strcmp(prev_cmd, "objects") == 0) {
664 if (strcmp(cmd, "expire") == 0)
665 return OPT_OBJECTS_EXPIRE;
666 } else if (strcmp(prev_cmd, "olh") == 0) {
667 if (strcmp(cmd, "get") == 0)
669 if (strcmp(cmd, "readlog") == 0)
670 return OPT_OLH_READLOG;
671 } else if (strcmp(prev_cmd, "bi") == 0) {
672 if (strcmp(cmd, "get") == 0)
674 if (strcmp(cmd, "put") == 0)
676 if (strcmp(cmd, "list") == 0)
678 if (strcmp(cmd, "purge") == 0)
680 } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "global") == 0) &&
681 (strcmp(prev_cmd, "quota") == 0)) {
682 if (strcmp(cmd, "get") == 0)
683 return OPT_GLOBAL_QUOTA_GET;
684 if (strcmp(cmd, "set") == 0)
685 return OPT_GLOBAL_QUOTA_SET;
686 if (strcmp(cmd, "enable") == 0)
687 return OPT_GLOBAL_QUOTA_ENABLE;
688 if (strcmp(cmd, "disable") == 0)
689 return OPT_GLOBAL_QUOTA_DISABLE;
690 } else if (strcmp(prev_cmd, "period") == 0) {
691 if (strcmp(cmd, "delete") == 0)
692 return OPT_PERIOD_DELETE;
693 if (strcmp(cmd, "get") == 0)
694 return OPT_PERIOD_GET;
695 if (strcmp(cmd, "get-current") == 0)
696 return OPT_PERIOD_GET_CURRENT;
697 if (strcmp(cmd, "pull") == 0)
698 return OPT_PERIOD_PULL;
699 if (strcmp(cmd, "push") == 0)
700 return OPT_PERIOD_PUSH;
701 if (strcmp(cmd, "list") == 0)
702 return OPT_PERIOD_LIST;
703 if (strcmp(cmd, "update") == 0)
704 return OPT_PERIOD_UPDATE;
705 if (strcmp(cmd, "commit") == 0)
706 return OPT_PERIOD_COMMIT;
707 } else if (strcmp(prev_cmd, "realm") == 0) {
708 if (strcmp(cmd, "create") == 0)
709 return OPT_REALM_CREATE;
710 if (strcmp(cmd, "delete") == 0)
711 return OPT_REALM_DELETE;
712 if (strcmp(cmd, "get") == 0)
713 return OPT_REALM_GET;
714 if (strcmp(cmd, "get-default") == 0)
715 return OPT_REALM_GET_DEFAULT;
716 if (strcmp(cmd, "list") == 0)
717 return OPT_REALM_LIST;
718 if (strcmp(cmd, "list-periods") == 0)
719 return OPT_REALM_LIST_PERIODS;
720 if (strcmp(cmd, "remove") == 0)
721 return OPT_REALM_REMOVE;
722 if (strcmp(cmd, "rename") == 0)
723 return OPT_REALM_RENAME;
724 if (strcmp(cmd, "set") == 0)
725 return OPT_REALM_SET;
726 if (strcmp(cmd, "default") == 0)
727 return OPT_REALM_DEFAULT;
728 if (strcmp(cmd, "pull") == 0)
729 return OPT_REALM_PULL;
730 } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "zonegroup") == 0) &&
731 (strcmp(prev_cmd, "placement") == 0)) {
732 if (strcmp(cmd, "add") == 0)
733 return OPT_ZONEGROUP_PLACEMENT_ADD;
734 if (strcmp(cmd, "modify") == 0)
735 return OPT_ZONEGROUP_PLACEMENT_MODIFY;
736 if (strcmp(cmd, "rm") == 0)
737 return OPT_ZONEGROUP_PLACEMENT_RM;
738 if (strcmp(cmd, "list") == 0)
739 return OPT_ZONEGROUP_PLACEMENT_LIST;
740 if (strcmp(cmd, "default") == 0)
741 return OPT_ZONEGROUP_PLACEMENT_DEFAULT;
742 } else if (strcmp(prev_cmd, "zonegroup") == 0) {
743 if (strcmp(cmd, "add") == 0)
744 return OPT_ZONEGROUP_ADD;
745 if (strcmp(cmd, "create")== 0)
746 return OPT_ZONEGROUP_CREATE;
747 if (strcmp(cmd, "default") == 0)
748 return OPT_ZONEGROUP_DEFAULT;
749 if (strcmp(cmd, "delete") == 0)
750 return OPT_ZONEGROUP_DELETE;
751 if (strcmp(cmd, "get") == 0)
752 return OPT_ZONEGROUP_GET;
753 if (strcmp(cmd, "modify") == 0)
754 return OPT_ZONEGROUP_MODIFY;
755 if (strcmp(cmd, "list") == 0)
756 return OPT_ZONEGROUP_LIST;
757 if (strcmp(cmd, "set") == 0)
758 return OPT_ZONEGROUP_SET;
759 if (strcmp(cmd, "remove") == 0)
760 return OPT_ZONEGROUP_REMOVE;
761 if (strcmp(cmd, "rename") == 0)
762 return OPT_ZONEGROUP_RENAME;
763 } else if (strcmp(prev_cmd, "quota") == 0) {
764 if (strcmp(cmd, "set") == 0)
765 return OPT_QUOTA_SET;
766 if (strcmp(cmd, "enable") == 0)
767 return OPT_QUOTA_ENABLE;
768 if (strcmp(cmd, "disable") == 0)
769 return OPT_QUOTA_DISABLE;
770 } else if (strcmp(prev_cmd, "zonegroups") == 0) {
771 if (strcmp(cmd, "list") == 0)
772 return OPT_ZONEGROUP_LIST;
773 } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "zone") == 0) &&
774 (strcmp(prev_cmd, "placement") == 0)) {
775 if (strcmp(cmd, "add") == 0)
776 return OPT_ZONE_PLACEMENT_ADD;
777 if (strcmp(cmd, "modify") == 0)
778 return OPT_ZONE_PLACEMENT_MODIFY;
779 if (strcmp(cmd, "rm") == 0)
780 return OPT_ZONE_PLACEMENT_RM;
781 if (strcmp(cmd, "list") == 0)
782 return OPT_ZONE_PLACEMENT_LIST;
783 } else if (strcmp(prev_cmd, "zone") == 0) {
784 if (strcmp(cmd, "delete") == 0)
785 return OPT_ZONE_DELETE;
786 if (strcmp(cmd, "create") == 0)
787 return OPT_ZONE_CREATE;
788 if (strcmp(cmd, "get") == 0)
790 if (strcmp(cmd, "set") == 0)
792 if (strcmp(cmd, "list") == 0)
793 return OPT_ZONE_LIST;
794 if (strcmp(cmd, "modify") == 0)
795 return OPT_ZONE_MODIFY;
796 if (strcmp(cmd, "rename") == 0)
797 return OPT_ZONE_RENAME;
798 if (strcmp(cmd, "default") == 0)
799 return OPT_ZONE_DEFAULT;
800 } else if (strcmp(prev_cmd, "zones") == 0) {
801 if (strcmp(cmd, "list") == 0)
802 return OPT_ZONE_LIST;
803 } else if (strcmp(prev_cmd, "gc") == 0) {
804 if (strcmp(cmd, "list") == 0)
806 if (strcmp(cmd, "process") == 0)
807 return OPT_GC_PROCESS;
808 } else if (strcmp(prev_cmd, "lc") == 0) {
809 if (strcmp(cmd, "list") == 0)
811 if (strcmp(cmd, "process") == 0)
812 return OPT_LC_PROCESS;
813 } else if (strcmp(prev_cmd, "orphans") == 0) {
814 if (strcmp(cmd, "find") == 0)
815 return OPT_ORPHANS_FIND;
816 if (strcmp(cmd, "finish") == 0)
817 return OPT_ORPHANS_FINISH;
818 if (strcmp(cmd, "list-jobs") == 0)
819 return OPT_ORPHANS_LIST_JOBS;
820 } else if (strcmp(prev_cmd, "metadata") == 0) {
821 if (strcmp(cmd, "get") == 0)
822 return OPT_METADATA_GET;
823 if (strcmp(cmd, "put") == 0)
824 return OPT_METADATA_PUT;
825 if (strcmp(cmd, "rm") == 0)
826 return OPT_METADATA_RM;
827 if (strcmp(cmd, "list") == 0)
828 return OPT_METADATA_LIST;
829 if (strcmp(cmd, "sync") == 0) {
833 } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "metadata") == 0) &&
834 (strcmp(prev_cmd, "sync") == 0)) {
835 if (strcmp(cmd, "status") == 0)
836 return OPT_METADATA_SYNC_STATUS;
837 if (strcmp(cmd, "init") == 0)
838 return OPT_METADATA_SYNC_INIT;
839 if (strcmp(cmd, "run") == 0)
840 return OPT_METADATA_SYNC_RUN;
841 } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "sync") == 0) &&
842 (strcmp(prev_cmd, "error") == 0)) {
843 if (strcmp(cmd, "list") == 0)
844 return OPT_SYNC_ERROR_LIST;
845 } else if (strcmp(prev_cmd, "mdlog") == 0) {
846 if (strcmp(cmd, "list") == 0)
847 return OPT_MDLOG_LIST;
848 if (strcmp(cmd, "autotrim") == 0)
849 return OPT_MDLOG_AUTOTRIM;
850 if (strcmp(cmd, "trim") == 0)
851 return OPT_MDLOG_TRIM;
852 if (strcmp(cmd, "fetch") == 0)
853 return OPT_MDLOG_FETCH;
854 if (strcmp(cmd, "status") == 0)
855 return OPT_MDLOG_STATUS;
856 } else if (strcmp(prev_cmd, "bilog") == 0) {
857 if (strcmp(cmd, "list") == 0)
858 return OPT_BILOG_LIST;
859 if (strcmp(cmd, "trim") == 0)
860 return OPT_BILOG_TRIM;
861 if (strcmp(cmd, "status") == 0)
862 return OPT_BILOG_STATUS;
863 } else if (strcmp(prev_cmd, "data") == 0) {
864 if (strcmp(cmd, "sync") == 0) {
868 } else if (strcmp(prev_cmd, "datalog") == 0) {
869 if (strcmp(cmd, "list") == 0)
870 return OPT_DATALOG_LIST;
871 if (strcmp(cmd, "trim") == 0)
872 return OPT_DATALOG_TRIM;
873 if (strcmp(cmd, "status") == 0)
874 return OPT_DATALOG_STATUS;
875 } else if ((prev_prev_cmd && strcmp(prev_prev_cmd, "data") == 0) &&
876 (strcmp(prev_cmd, "sync") == 0)) {
877 if (strcmp(cmd, "status") == 0)
878 return OPT_DATA_SYNC_STATUS;
879 if (strcmp(cmd, "init") == 0)
880 return OPT_DATA_SYNC_INIT;
881 if (strcmp(cmd, "run") == 0)
882 return OPT_DATA_SYNC_RUN;
883 } else if (strcmp(prev_cmd, "opstate") == 0) {
884 if (strcmp(cmd, "list") == 0)
885 return OPT_OPSTATE_LIST;
886 if (strcmp(cmd, "set") == 0)
887 return OPT_OPSTATE_SET;
888 if (strcmp(cmd, "renew") == 0)
889 return OPT_OPSTATE_RENEW;
890 if (strcmp(cmd, "rm") == 0)
891 return OPT_OPSTATE_RM;
892 } else if (strcmp(prev_cmd, "replicalog") == 0) {
893 if (strcmp(cmd, "get") == 0)
894 return OPT_REPLICALOG_GET;
895 if (strcmp(cmd, "update") == 0)
896 return OPT_REPLICALOG_UPDATE;
897 if (strcmp(cmd, "delete") == 0)
898 return OPT_REPLICALOG_DELETE;
899 } else if (strcmp(prev_cmd, "sync") == 0) {
900 if (strcmp(cmd, "status") == 0)
901 return OPT_SYNC_STATUS;
902 } else if (strcmp(prev_cmd, "role") == 0) {
903 if (strcmp(cmd, "create") == 0)
904 return OPT_ROLE_CREATE;
905 if (strcmp(cmd, "delete") == 0)
906 return OPT_ROLE_DELETE;
907 if (strcmp(cmd, "get") == 0)
909 if (strcmp(cmd, "modify") == 0)
910 return OPT_ROLE_MODIFY;
911 if (strcmp(cmd, "list") == 0)
912 return OPT_ROLE_LIST;
913 } else if (strcmp(prev_cmd, "role-policy") == 0) {
914 if (strcmp(cmd, "put") == 0)
915 return OPT_ROLE_POLICY_PUT;
916 if (strcmp(cmd, "list") == 0)
917 return OPT_ROLE_POLICY_LIST;
918 if (strcmp(cmd, "get") == 0)
919 return OPT_ROLE_POLICY_GET;
920 if (strcmp(cmd, "delete") == 0)
921 return OPT_ROLE_POLICY_DELETE;
922 } else if (strcmp(prev_cmd, "reshard") == 0) {
923 if (strcmp(cmd, "bucket") == 0)
924 return OPT_BUCKET_RESHARD;
925 if (strcmp(cmd, "add") == 0)
926 return OPT_RESHARD_ADD;
927 if (strcmp(cmd, "list") == 0)
928 return OPT_RESHARD_LIST;
929 if (strcmp(cmd, "status") == 0)
930 return OPT_RESHARD_STATUS;
931 if (strcmp(cmd, "execute") == 0)
932 return OPT_RESHARD_PROCESS;
933 if (strcmp(cmd, "cancel") == 0)
934 return OPT_RESHARD_CANCEL;
940 enum ReplicaLogType {
941 ReplicaLog_Invalid = 0,
947 ReplicaLogType get_replicalog_type(const string& name) {
948 if (name == "md" || name == "meta" || name == "metadata")
949 return ReplicaLog_Metadata;
951 return ReplicaLog_Data;
952 if (name == "bucket")
953 return ReplicaLog_Bucket;
955 return ReplicaLog_Invalid;
958 BIIndexType get_bi_index_type(const string& type_str) {
959 if (type_str == "plain")
961 if (type_str == "instance")
963 if (type_str == "olh")
969 void dump_bi_entry(bufferlist& bl, BIIndexType index_type, Formatter *formatter)
971 bufferlist::iterator iter = bl.begin();
972 switch (index_type) {
976 rgw_bucket_dir_entry entry;
977 ::decode(entry, iter);
978 encode_json("entry", entry, formatter);
983 rgw_bucket_olh_entry entry;
984 ::decode(entry, iter);
985 encode_json("entry", entry, formatter);
994 static void show_user_info(RGWUserInfo& info, Formatter *formatter)
996 encode_json("user_info", info, formatter);
997 formatter->flush(cout);
1001 static void show_perm_policy(string perm_policy, Formatter* formatter)
1003 formatter->open_object_section("role");
1004 formatter->dump_string("Permission policy", perm_policy);
1005 formatter->close_section();
1006 formatter->flush(cout);
1009 static void show_policy_names(std::vector<string> policy_names, Formatter* formatter)
1011 formatter->open_array_section("PolicyNames");
1012 for (const auto& it : policy_names) {
1013 formatter->dump_string("policyname", it);
1015 formatter->close_section();
1016 formatter->flush(cout);
1019 static void show_role_info(RGWRole& role, Formatter* formatter)
1021 formatter->open_object_section("role");
1022 role.dump(formatter);
1023 formatter->close_section();
1024 formatter->flush(cout);
1027 static void show_roles_info(vector<RGWRole>& roles, Formatter* formatter)
1029 formatter->open_array_section("Roles");
1030 for (const auto& it : roles) {
1031 formatter->open_object_section("role");
1033 formatter->close_section();
1035 formatter->close_section();
1036 formatter->flush(cout);
1039 class StoreDestructor {
1042 explicit StoreDestructor(RGWRados *_s) : store(_s) {}
1043 ~StoreDestructor() {
1044 RGWStoreManager::close_storage(store);
1048 static int init_bucket(const string& tenant_name, const string& bucket_name, const string& bucket_id,
1049 RGWBucketInfo& bucket_info, rgw_bucket& bucket, map<string, bufferlist> *pattrs = nullptr)
1051 if (!bucket_name.empty()) {
1052 RGWObjectCtx obj_ctx(store);
1054 if (bucket_id.empty()) {
1055 r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, nullptr, pattrs);
1057 string bucket_instance_id = bucket_name + ":" + bucket_id;
1058 r = store->get_bucket_instance_info(obj_ctx, bucket_instance_id, bucket_info, NULL, pattrs);
1061 cerr << "could not get bucket info for bucket=" << bucket_name << std::endl;
1064 bucket = bucket_info.bucket;
1069 static int read_input(const string& infile, bufferlist& bl)
1072 if (infile.size()) {
1073 fd = open(infile.c_str(), O_RDONLY);
1076 cerr << "error reading input file " << infile << std::endl;
1081 #define READ_CHUNK 8196
1086 char buf[READ_CHUNK];
1088 r = safe_read(fd, buf, READ_CHUNK);
1091 cerr << "error while reading input" << std::endl;
1099 if (infile.size()) {
1106 static int read_decode_json(const string& infile, T& t)
1109 int ret = read_input(infile, bl);
1111 cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
1115 if (!p.parse(bl.c_str(), bl.length())) {
1116 cout << "failed to parse JSON" << std::endl;
1121 decode_json_obj(t, &p);
1122 } catch (JSONDecoder::err& e) {
1123 cout << "failed to decode JSON input: " << e.message << std::endl;
1129 template <class T, class K>
1130 static int read_decode_json(const string& infile, T& t, K *k)
1133 int ret = read_input(infile, bl);
1135 cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
1139 if (!p.parse(bl.c_str(), bl.length())) {
1140 cout << "failed to parse JSON" << std::endl;
1145 t.decode_json(&p, k);
1146 } catch (JSONDecoder::err& e) {
1147 cout << "failed to decode JSON input: " << e.message << std::endl;
1153 static int parse_date_str(const string& date_str, utime_t& ut)
1158 if (!date_str.empty()) {
1159 int ret = utime_t::parse_date(date_str, &epoch, &nsec);
1161 cerr << "ERROR: failed to parse date: " << date_str << std::endl;
1166 ut = utime_t(epoch, nsec);
1172 static bool decode_dump(const char *field_name, bufferlist& bl, Formatter *f)
1176 bufferlist::iterator iter = bl.begin();
1180 } catch (buffer::error& err) {
1184 encode_json(field_name, t, f);
1189 static bool dump_string(const char *field_name, bufferlist& bl, Formatter *f)
1192 if (bl.length() > 0) {
1193 val.assign(bl.c_str());
1195 f->dump_string(field_name, val);
1200 void set_quota_info(RGWQuotaInfo& quota, int opt_cmd, int64_t max_size, int64_t max_objects,
1201 bool have_max_size, bool have_max_objects)
1204 case OPT_QUOTA_ENABLE:
1205 case OPT_GLOBAL_QUOTA_ENABLE:
1206 quota.enabled = true;
1208 // falling through on purpose
1211 case OPT_GLOBAL_QUOTA_SET:
1212 if (have_max_objects) {
1213 if (max_objects < 0) {
1214 quota.max_objects = -1;
1216 quota.max_objects = max_objects;
1219 if (have_max_size) {
1221 quota.max_size = -1;
1223 quota.max_size = rgw_rounded_kb(max_size) * 1024;
1227 case OPT_QUOTA_DISABLE:
1228 case OPT_GLOBAL_QUOTA_DISABLE:
1229 quota.enabled = false;
1234 int set_bucket_quota(RGWRados *store, int opt_cmd,
1235 const string& tenant_name, const string& bucket_name,
1236 int64_t max_size, int64_t max_objects,
1237 bool have_max_size, bool have_max_objects)
1239 RGWBucketInfo bucket_info;
1240 map<string, bufferlist> attrs;
1241 RGWObjectCtx obj_ctx(store);
1242 int r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, &attrs);
1244 cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
1248 set_quota_info(bucket_info.quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
1250 r = store->put_bucket_instance_info(bucket_info, false, real_time(), &attrs);
1252 cerr << "ERROR: failed writing bucket instance info: " << cpp_strerror(-r) << std::endl;
1258 int set_user_bucket_quota(int opt_cmd, RGWUser& user, RGWUserAdminOpState& op_state, int64_t max_size, int64_t max_objects,
1259 bool have_max_size, bool have_max_objects)
1261 RGWUserInfo& user_info = op_state.get_user_info();
1263 set_quota_info(user_info.bucket_quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
1265 op_state.set_bucket_quota(user_info.bucket_quota);
1268 int r = user.modify(op_state, &err);
1270 cerr << "ERROR: failed updating user info: " << cpp_strerror(-r) << ": " << err << std::endl;
1276 int set_user_quota(int opt_cmd, RGWUser& user, RGWUserAdminOpState& op_state, int64_t max_size, int64_t max_objects,
1277 bool have_max_size, bool have_max_objects)
1279 RGWUserInfo& user_info = op_state.get_user_info();
1281 set_quota_info(user_info.user_quota, opt_cmd, max_size, max_objects, have_max_size, have_max_objects);
1283 op_state.set_user_quota(user_info.user_quota);
1286 int r = user.modify(op_state, &err);
1288 cerr << "ERROR: failed updating user info: " << cpp_strerror(-r) << ": " << err << std::endl;
1294 static bool bucket_object_check_filter(const string& name)
1297 string ns; /* empty namespace */
1298 return rgw_obj_key::oid_to_key_in_ns(name, &k, ns);
1301 int check_min_obj_stripe_size(RGWRados *store, RGWBucketInfo& bucket_info, rgw_obj& obj, uint64_t min_stripe_size, bool *need_rewrite)
1303 map<string, bufferlist> attrs;
1306 RGWObjectCtx obj_ctx(store);
1307 RGWRados::Object op_target(store, bucket_info, obj_ctx, obj);
1308 RGWRados::Object::Read read_op(&op_target);
1310 read_op.params.attrs = &attrs;
1311 read_op.params.obj_size = &obj_size;
1313 int ret = read_op.prepare();
1315 lderr(store->ctx()) << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << dendl;
1319 map<string, bufferlist>::iterator iter;
1320 iter = attrs.find(RGW_ATTR_MANIFEST);
1321 if (iter == attrs.end()) {
1322 *need_rewrite = (obj_size >= min_stripe_size);
1326 RGWObjManifest manifest;
1329 bufferlist& bl = iter->second;
1330 bufferlist::iterator biter = bl.begin();
1331 ::decode(manifest, biter);
1332 } catch (buffer::error& err) {
1333 ldout(store->ctx(), 0) << "ERROR: failed to decode manifest" << dendl;
1337 map<uint64_t, RGWObjManifestPart>& objs = manifest.get_explicit_objs();
1338 map<uint64_t, RGWObjManifestPart>::iterator oiter;
1339 for (oiter = objs.begin(); oiter != objs.end(); ++oiter) {
1340 RGWObjManifestPart& part = oiter->second;
1342 if (part.size >= min_stripe_size) {
1343 *need_rewrite = true;
1347 *need_rewrite = false;
1353 int check_obj_locator_underscore(RGWBucketInfo& bucket_info, rgw_obj& obj, rgw_obj_key& key, bool fix, bool remove_bad, Formatter *f) {
1354 f->open_object_section("object");
1355 f->open_object_section("key");
1356 f->dump_string("type", "head");
1357 f->dump_string("name", key.name);
1358 f->dump_string("instance", key.instance);
1364 get_obj_bucket_and_oid_loc(obj, oid, locator);
1366 f->dump_string("oid", oid);
1367 f->dump_string("locator", locator);
1370 RGWObjectCtx obj_ctx(store);
1372 RGWRados::Object op_target(store, bucket_info, obj_ctx, obj);
1373 RGWRados::Object::Read read_op(&op_target);
1375 int ret = read_op.prepare();
1376 bool needs_fixing = (ret == -ENOENT);
1378 f->dump_bool("needs_fixing", needs_fixing);
1380 string status = (needs_fixing ? "needs_fixing" : "ok");
1382 if ((needs_fixing || remove_bad) && fix) {
1383 ret = store->fix_head_obj_locator(bucket_info, needs_fixing, remove_bad, key);
1385 cerr << "ERROR: fix_head_object_locator() returned ret=" << ret << std::endl;
1392 f->dump_string("status", status);
1399 int check_obj_tail_locator_underscore(RGWBucketInfo& bucket_info, rgw_obj& obj, rgw_obj_key& key, bool fix, Formatter *f) {
1400 f->open_object_section("object");
1401 f->open_object_section("key");
1402 f->dump_string("type", "tail");
1403 f->dump_string("name", key.name);
1404 f->dump_string("instance", key.instance);
1410 int ret = store->fix_tail_obj_locator(bucket_info, key, fix, &needs_fixing);
1412 cerr << "ERROR: fix_tail_object_locator_underscore() returned ret=" << ret << std::endl;
1415 status = (needs_fixing && !fix ? "needs_fixing" : "ok");
1418 f->dump_bool("needs_fixing", needs_fixing);
1419 f->dump_string("status", status);
1426 int do_check_object_locator(const string& tenant_name, const string& bucket_name,
1427 bool fix, bool remove_bad, Formatter *f)
1429 if (remove_bad && !fix) {
1430 cerr << "ERROR: can't have remove_bad specified without fix" << std::endl;
1434 RGWBucketInfo bucket_info;
1438 f->open_object_section("bucket");
1439 f->dump_string("bucket", bucket_name);
1440 int ret = init_bucket(tenant_name, bucket_name, bucket_id, bucket_info, bucket);
1442 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
1448 int max_entries = 1000;
1452 vector<rgw_bucket_dir_entry> result;
1453 map<string, bool> common_prefixes;
1456 RGWRados::Bucket target(store, bucket_info);
1457 RGWRados::Bucket::List list_op(&target);
1461 list_op.params.prefix = prefix;
1462 list_op.params.delim = delim;
1463 list_op.params.marker = rgw_obj_key(marker);
1464 list_op.params.ns = ns;
1465 list_op.params.enforce_ns = true;
1466 list_op.params.list_versions = true;
1468 f->open_array_section("check_objects");
1470 ret = list_op.list_objects(max_entries - count, &result, &common_prefixes, &truncated);
1472 cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl;
1476 count += result.size();
1478 for (vector<rgw_bucket_dir_entry>::iterator iter = result.begin(); iter != result.end(); ++iter) {
1479 rgw_obj_key key = iter->key;
1480 rgw_obj obj(bucket, key);
1482 if (key.name[0] == '_') {
1483 ret = check_obj_locator_underscore(bucket_info, obj, key, fix, remove_bad, f);
1486 ret = check_obj_tail_locator_underscore(bucket_info, obj, key, fix, f);
1491 } while (truncated && count < max_entries);
1500 int set_bucket_sync_enabled(RGWRados *store, int opt_cmd, const string& tenant_name, const string& bucket_name)
1502 RGWBucketInfo bucket_info;
1503 map<string, bufferlist> attrs;
1504 RGWObjectCtx obj_ctx(store);
1506 int r = store->get_bucket_info(obj_ctx, tenant_name, bucket_name, bucket_info, NULL, &attrs);
1508 cerr << "could not get bucket info for bucket=" << bucket_name << ": " << cpp_strerror(-r) << std::endl;
1512 if (opt_cmd == OPT_BUCKET_SYNC_ENABLE) {
1513 bucket_info.flags &= ~BUCKET_DATASYNC_DISABLED;
1514 } else if (opt_cmd == OPT_BUCKET_SYNC_DISABLE) {
1515 bucket_info.flags |= BUCKET_DATASYNC_DISABLED;
1518 r = store->put_bucket_instance_info(bucket_info, false, real_time(), &attrs);
1520 cerr << "ERROR: failed writing bucket instance info: " << cpp_strerror(-r) << std::endl;
1524 int shards_num = bucket_info.num_shards? bucket_info.num_shards : 1;
1525 int shard_id = bucket_info.num_shards? 0 : -1;
1527 if (opt_cmd == OPT_BUCKET_SYNC_DISABLE) {
1528 r = store->stop_bi_log_entries(bucket_info, -1);
1530 lderr(store->ctx()) << "ERROR: failed writing stop bilog" << dendl;
1534 r = store->resync_bi_log_entries(bucket_info, -1);
1536 lderr(store->ctx()) << "ERROR: failed writing resync bilog" << dendl;
1541 for (int i = 0; i < shards_num; ++i, ++shard_id) {
1542 r = store->data_log->add_entry(bucket_info.bucket, shard_id);
1544 lderr(store->ctx()) << "ERROR: failed writing data log" << dendl;
1553 /// search for a matching zone/zonegroup id and return a connection if found
1554 static boost::optional<RGWRESTConn> get_remote_conn(RGWRados *store,
1555 const RGWZoneGroup& zonegroup,
1556 const std::string& remote)
1558 boost::optional<RGWRESTConn> conn;
1559 if (remote == zonegroup.get_id()) {
1560 conn.emplace(store->ctx(), store, remote, zonegroup.endpoints);
1562 for (const auto& z : zonegroup.zones) {
1563 const auto& zone = z.second;
1564 if (remote == zone.id) {
1565 conn.emplace(store->ctx(), store, remote, zone.endpoints);
1573 /// search each zonegroup for a connection
1574 static boost::optional<RGWRESTConn> get_remote_conn(RGWRados *store,
1575 const RGWPeriodMap& period_map,
1576 const std::string& remote)
1578 boost::optional<RGWRESTConn> conn;
1579 for (const auto& zg : period_map.zonegroups) {
1580 conn = get_remote_conn(store, zg.second, remote);
1588 // we expect a very small response
1589 static constexpr size_t MAX_REST_RESPONSE = 128 * 1024;
1591 static int send_to_remote_gateway(RGWRESTConn* conn, req_info& info,
1592 bufferlist& in_data, JSONParser& parser)
1598 ceph::bufferlist response;
1600 int ret = conn->forward(user, info, nullptr, MAX_REST_RESPONSE, &in_data, &response);
1602 int parse_ret = parser.parse(response.c_str(), response.length());
1603 if (parse_ret < 0) {
1604 cerr << "failed to parse response" << std::endl;
1610 static int send_to_url(const string& url, const string& access,
1611 const string& secret, req_info& info,
1612 bufferlist& in_data, JSONParser& parser)
1614 if (access.empty() || secret.empty()) {
1615 cerr << "An --access-key and --secret must be provided with --url." << std::endl;
1623 RGWRESTSimpleRequest req(g_ceph_context, url, NULL, ¶ms);
1625 bufferlist response;
1626 int ret = req.forward_request(key, info, MAX_REST_RESPONSE, &in_data, &response);
1628 int parse_ret = parser.parse(response.c_str(), response.length());
1629 if (parse_ret < 0) {
1630 cout << "failed to parse response" << std::endl;
1636 static int send_to_remote_or_url(RGWRESTConn *conn, const string& url,
1637 const string& access, const string& secret,
1638 req_info& info, bufferlist& in_data,
1642 return send_to_remote_gateway(conn, info, in_data, parser);
1644 return send_to_url(url, access, secret, info, in_data, parser);
1647 static int commit_period(RGWRealm& realm, RGWPeriod& period,
1648 string remote, const string& url,
1649 const string& access, const string& secret,
1652 const string& master_zone = period.get_master_zone();
1653 if (master_zone.empty()) {
1654 cerr << "cannot commit period: period does not have a master zone of a master zonegroup" << std::endl;
1657 // are we the period's master zone?
1658 if (store->get_zone_params().get_id() == master_zone) {
1659 // read the current period
1660 RGWPeriod current_period;
1661 int ret = current_period.init(g_ceph_context, store, realm.get_id());
1663 cerr << "Error initializing current period: "
1664 << cpp_strerror(-ret) << std::endl;
1667 // the master zone can commit locally
1668 ret = period.commit(realm, current_period, cerr, force);
1670 cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl;
1675 if (remote.empty() && url.empty()) {
1676 // use the new master zone's connection
1677 remote = master_zone;
1678 cout << "Sending period to new master zone " << remote << std::endl;
1680 boost::optional<RGWRESTConn> conn;
1681 RGWRESTConn *remote_conn = nullptr;
1682 if (!remote.empty()) {
1683 conn = get_remote_conn(store, period.get_map(), remote);
1685 cerr << "failed to find a zone or zonegroup for remote "
1686 << remote << std::endl;
1689 remote_conn = &*conn;
1692 // push period to the master with an empty period id
1696 req_info info(g_ceph_context, &env);
1697 info.method = "POST";
1698 info.request_uri = "/admin/realm/period";
1700 // json format into a bufferlist
1701 JSONFormatter jf(false);
1702 encode_json("period", period, &jf);
1707 int ret = send_to_remote_or_url(remote_conn, url, access, secret, info, bl, p);
1709 cerr << "request failed: " << cpp_strerror(-ret) << std::endl;
1711 // did we parse an error message?
1712 auto message = p.find_obj("Message");
1714 cerr << "Reason: " << message->get_data() << std::endl;
1719 // decode the response and store it back
1721 decode_json_obj(period, &p);
1722 } catch (JSONDecoder::err& e) {
1723 cout << "failed to decode JSON input: " << e.message << std::endl;
1726 if (period.get_id().empty()) {
1727 cerr << "Period commit got back an empty period id" << std::endl;
1730 // the master zone gave us back the period that it committed, so it's
1731 // safe to save it as our latest epoch
1732 ret = period.store_info(false);
1734 cerr << "Error storing committed period " << period.get_id() << ": "
1735 << cpp_strerror(ret) << std::endl;
1738 ret = period.set_latest_epoch(period.get_epoch());
1740 cerr << "Error updating period epoch: " << cpp_strerror(ret) << std::endl;
1743 ret = period.reflect();
1745 cerr << "Error updating local objects: " << cpp_strerror(ret) << std::endl;
1748 realm.notify_new_period(period);
1752 static int update_period(const string& realm_id, const string& realm_name,
1753 const string& period_id, const string& period_epoch,
1754 bool commit, const string& remote, const string& url,
1755 const string& access, const string& secret,
1756 Formatter *formatter, bool force)
1758 RGWRealm realm(realm_id, realm_name);
1759 int ret = realm.init(g_ceph_context, store);
1761 cerr << "Error initializing realm " << cpp_strerror(-ret) << std::endl;
1765 if (!period_epoch.empty()) {
1766 epoch = atoi(period_epoch.c_str());
1768 RGWPeriod period(period_id, epoch);
1769 ret = period.init(g_ceph_context, store, realm.get_id());
1771 cerr << "period init failed: " << cpp_strerror(-ret) << std::endl;
1775 ret = period.update();
1777 // Dropping the error message here, as both the ret codes were handled in
1781 ret = period.store_info(false);
1783 cerr << "failed to store period: " << cpp_strerror(-ret) << std::endl;
1787 ret = commit_period(realm, period, remote, url, access, secret, force);
1789 cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl;
1793 encode_json("period", period, formatter);
1794 formatter->flush(cout);
1799 static int init_bucket_for_sync(const string& tenant, const string& bucket_name,
1800 const string& bucket_id, rgw_bucket& bucket)
1802 RGWBucketInfo bucket_info;
1804 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
1806 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
1813 static int do_period_pull(RGWRESTConn *remote_conn, const string& url,
1814 const string& access_key, const string& secret_key,
1815 const string& realm_id, const string& realm_name,
1816 const string& period_id, const string& period_epoch,
1820 req_info info(g_ceph_context, &env);
1821 info.method = "GET";
1822 info.request_uri = "/admin/realm/period";
1824 map<string, string> ¶ms = info.args.get_params();
1825 if (!realm_id.empty())
1826 params["realm_id"] = realm_id;
1827 if (!realm_name.empty())
1828 params["realm_name"] = realm_name;
1829 if (!period_id.empty())
1830 params["period_id"] = period_id;
1831 if (!period_epoch.empty())
1832 params["epoch"] = period_epoch;
1836 int ret = send_to_remote_or_url(remote_conn, url, access_key, secret_key,
1839 cerr << "request failed: " << cpp_strerror(-ret) << std::endl;
1842 ret = period->init(g_ceph_context, store, false);
1844 cerr << "faile to init period " << cpp_strerror(-ret) << std::endl;
1848 decode_json_obj(*period, &p);
1849 } catch (JSONDecoder::err& e) {
1850 cout << "failed to decode JSON input: " << e.message << std::endl;
1853 ret = period->store_info(false);
1855 cerr << "Error storing period " << period->get_id() << ": " << cpp_strerror(ret) << std::endl;
1857 // store latest epoch (ignore errors)
1858 period->update_latest_epoch(period->get_epoch());
1862 static int read_current_period_id(RGWRados* store, const std::string& realm_id,
1863 const std::string& realm_name,
1864 std::string* period_id)
1866 RGWRealm realm(realm_id, realm_name);
1867 int ret = realm.init(g_ceph_context, store);
1869 std::cerr << "failed to read realm: " << cpp_strerror(-ret) << std::endl;
1872 *period_id = realm.get_current_period();
1876 void flush_ss(stringstream& ss, list<string>& l)
1878 if (!ss.str().empty()) {
1879 l.push_back(ss.str());
1884 stringstream& push_ss(stringstream& ss, list<string>& l, int tab = 0)
1888 ss << setw(tab) << "" << setw(1);
1893 static void get_md_sync_status(list<string>& status)
1895 RGWMetaSyncStatusManager sync(store, store->get_async_rados());
1897 int ret = sync.init();
1899 status.push_back(string("failed to retrieve sync info: sync.init() failed: ") + cpp_strerror(-ret));
1903 rgw_meta_sync_status sync_status;
1904 ret = sync.read_sync_status(&sync_status);
1906 status.push_back(string("failed to read sync status: ") + cpp_strerror(-ret));
1911 switch (sync_status.sync_info.state) {
1912 case rgw_meta_sync_info::StateInit:
1913 status_str = "init";
1915 case rgw_meta_sync_info::StateBuildingFullSyncMaps:
1916 status_str = "preparing for full sync";
1918 case rgw_meta_sync_info::StateSync:
1919 status_str = "syncing";
1922 status_str = "unknown";
1925 status.push_back(status_str);
1927 uint64_t full_total = 0;
1928 uint64_t full_complete = 0;
1932 int total_shards = 0;
1934 for (auto marker_iter : sync_status.sync_markers) {
1935 full_total += marker_iter.second.total_entries;
1937 if (marker_iter.second.state == rgw_meta_sync_marker::SyncState::FullSync) {
1939 full_complete += marker_iter.second.pos;
1941 full_complete += marker_iter.second.total_entries;
1943 if (marker_iter.second.state == rgw_meta_sync_marker::SyncState::IncrementalSync) {
1949 push_ss(ss, status) << "full sync: " << num_full << "/" << total_shards << " shards";
1952 push_ss(ss, status) << "full sync: " << full_total - full_complete << " entries to sync";
1955 push_ss(ss, status) << "incremental sync: " << num_inc << "/" << total_shards << " shards";
1957 rgw_mdlog_info log_info;
1958 ret = sync.read_log_info(&log_info);
1960 status.push_back(string("failed to fetch local sync status: ") + cpp_strerror(-ret));
1964 map<int, RGWMetadataLogInfo> master_shards_info;
1965 string master_period = store->get_current_period_id();
1967 ret = sync.read_master_log_shards_info(master_period, &master_shards_info);
1969 status.push_back(string("failed to fetch master sync status: ") + cpp_strerror(-ret));
1973 map<int, string> shards_behind;
1974 if (sync_status.sync_info.period != master_period) {
1975 status.push_back(string("master is on a different period: master_period=" +
1976 master_period + " local_period=" + sync_status.sync_info.period));
1978 for (auto local_iter : sync_status.sync_markers) {
1979 int shard_id = local_iter.first;
1980 auto iter = master_shards_info.find(shard_id);
1982 if (iter == master_shards_info.end()) {
1984 derr << "ERROR: could not find remote sync shard status for shard_id=" << shard_id << dendl;
1987 auto master_marker = iter->second.marker;
1988 if (local_iter.second.state == rgw_meta_sync_marker::SyncState::IncrementalSync &&
1989 master_marker > local_iter.second.marker) {
1990 shards_behind[shard_id] = local_iter.second.marker;
1995 int total_behind = shards_behind.size() + (sync_status.sync_info.num_shards - num_inc);
1996 if (total_behind == 0) {
1997 push_ss(ss, status) << "metadata is caught up with master";
1999 push_ss(ss, status) << "metadata is behind on " << total_behind << " shards";
2001 map<int, rgw_mdlog_shard_data> master_pos;
2002 ret = sync.read_master_log_shards_next(sync_status.sync_info.period, shards_behind, &master_pos);
2004 derr << "ERROR: failed to fetch master next positions (" << cpp_strerror(-ret) << ")" << dendl;
2006 ceph::real_time oldest;
2007 for (auto iter : master_pos) {
2008 rgw_mdlog_shard_data& shard_data = iter.second;
2010 if (!shard_data.entries.empty()) {
2011 rgw_mdlog_entry& entry = shard_data.entries.front();
2012 if (ceph::real_clock::is_zero(oldest)) {
2013 oldest = entry.timestamp;
2014 } else if (!ceph::real_clock::is_zero(entry.timestamp) && entry.timestamp < oldest) {
2015 oldest = entry.timestamp;
2020 if (!ceph::real_clock::is_zero(oldest)) {
2021 push_ss(ss, status) << "oldest incremental change not applied: " << oldest;
2026 flush_ss(ss, status);
2029 static void get_data_sync_status(const string& source_zone, list<string>& status, int tab)
2033 auto ziter = store->zone_by_id.find(source_zone);
2034 if (ziter == store->zone_by_id.end()) {
2035 push_ss(ss, status, tab) << string("zone not found");
2036 flush_ss(ss, status);
2039 RGWZone& sz = ziter->second;
2041 if (!store->zone_syncs_from(store->get_zone(), sz)) {
2042 push_ss(ss, status, tab) << string("not syncing from zone");
2043 flush_ss(ss, status);
2046 RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone);
2048 int ret = sync.init();
2050 push_ss(ss, status, tab) << string("failed to retrieve sync info: ") + cpp_strerror(-ret);
2051 flush_ss(ss, status);
2055 rgw_data_sync_status sync_status;
2056 ret = sync.read_sync_status(&sync_status);
2057 if (ret < 0 && ret != -ENOENT) {
2058 push_ss(ss, status, tab) << string("failed read sync status: ") + cpp_strerror(-ret);
2063 switch (sync_status.sync_info.state) {
2064 case rgw_data_sync_info::StateInit:
2065 status_str = "init";
2067 case rgw_data_sync_info::StateBuildingFullSyncMaps:
2068 status_str = "preparing for full sync";
2070 case rgw_data_sync_info::StateSync:
2071 status_str = "syncing";
2074 status_str = "unknown";
2077 push_ss(ss, status, tab) << status_str;
2079 uint64_t full_total = 0;
2080 uint64_t full_complete = 0;
2084 int total_shards = 0;
2086 for (auto marker_iter : sync_status.sync_markers) {
2087 full_total += marker_iter.second.total_entries;
2089 if (marker_iter.second.state == rgw_data_sync_marker::SyncState::FullSync) {
2091 full_complete += marker_iter.second.pos;
2093 full_complete += marker_iter.second.total_entries;
2095 if (marker_iter.second.state == rgw_data_sync_marker::SyncState::IncrementalSync) {
2100 push_ss(ss, status, tab) << "full sync: " << num_full << "/" << total_shards << " shards";
2103 push_ss(ss, status, tab) << "full sync: " << full_total - full_complete << " buckets to sync";
2106 push_ss(ss, status, tab) << "incremental sync: " << num_inc << "/" << total_shards << " shards";
2108 rgw_datalog_info log_info;
2109 ret = sync.read_log_info(&log_info);
2111 push_ss(ss, status, tab) << string("failed to fetch local sync status: ") + cpp_strerror(-ret);
2116 map<int, RGWDataChangesLogInfo> source_shards_info;
2118 ret = sync.read_source_log_shards_info(&source_shards_info);
2120 push_ss(ss, status, tab) << string("failed to fetch source sync status: ") + cpp_strerror(-ret);
2124 map<int, string> shards_behind;
2126 for (auto local_iter : sync_status.sync_markers) {
2127 int shard_id = local_iter.first;
2128 auto iter = source_shards_info.find(shard_id);
2130 if (iter == source_shards_info.end()) {
2132 derr << "ERROR: could not find remote sync shard status for shard_id=" << shard_id << dendl;
2135 auto master_marker = iter->second.marker;
2136 if (local_iter.second.state == rgw_data_sync_marker::SyncState::IncrementalSync &&
2137 master_marker > local_iter.second.marker) {
2138 shards_behind[shard_id] = local_iter.second.marker;
2142 int total_behind = shards_behind.size() + (sync_status.sync_info.num_shards - num_inc);
2143 if (total_behind == 0) {
2144 push_ss(ss, status, tab) << "data is caught up with source";
2146 push_ss(ss, status, tab) << "data is behind on " << total_behind << " shards";
2148 map<int, rgw_datalog_shard_data> master_pos;
2149 ret = sync.read_source_log_shards_next(shards_behind, &master_pos);
2151 derr << "ERROR: failed to fetch next positions (" << cpp_strerror(-ret) << ")" << dendl;
2153 ceph::real_time oldest;
2154 for (auto iter : master_pos) {
2155 rgw_datalog_shard_data& shard_data = iter.second;
2157 if (!shard_data.entries.empty()) {
2158 rgw_datalog_entry& entry = shard_data.entries.front();
2159 if (ceph::real_clock::is_zero(oldest)) {
2160 oldest = entry.timestamp;
2161 } else if (!ceph::real_clock::is_zero(entry.timestamp) && entry.timestamp < oldest) {
2162 oldest = entry.timestamp;
2167 if (!ceph::real_clock::is_zero(oldest)) {
2168 push_ss(ss, status, tab) << "oldest incremental change not applied: " << oldest;
2173 flush_ss(ss, status);
2176 static void tab_dump(const string& header, int width, const list<string>& entries)
2180 for (auto e : entries) {
2181 cout << std::setw(width) << s << std::setw(1) << " " << e << std::endl;
2187 static void sync_status(Formatter *formatter)
2189 RGWRealm& realm = store->realm;
2190 RGWZoneGroup& zonegroup = store->get_zonegroup();
2191 RGWZone& zone = store->get_zone();
2195 cout << std::setw(width) << "realm" << std::setw(1) << " " << realm.get_id() << " (" << realm.get_name() << ")" << std::endl;
2196 cout << std::setw(width) << "zonegroup" << std::setw(1) << " " << zonegroup.get_id() << " (" << zonegroup.get_name() << ")" << std::endl;
2197 cout << std::setw(width) << "zone" << std::setw(1) << " " << zone.id << " (" << zone.name << ")" << std::endl;
2199 list<string> md_status;
2201 if (store->is_meta_master()) {
2202 md_status.push_back("no sync (zone is master)");
2204 get_md_sync_status(md_status);
2207 tab_dump("metadata sync", width, md_status);
2209 list<string> data_status;
2211 for (auto iter : store->zone_conn_map) {
2212 const string& source_id = iter.first;
2213 string source_str = "source: ";
2214 string s = source_str + source_id;
2215 auto siter = store->zone_by_id.find(source_id);
2216 if (siter != store->zone_by_id.end()) {
2217 s += string(" (") + siter->second.name + ")";
2219 data_status.push_back(s);
2220 get_data_sync_status(source_id, data_status, source_str.size());
2223 tab_dump("data sync", width, data_status);
2226 static void parse_tier_config_param(const string& s, map<string, string, ltstr_nocase>& out)
2229 get_str_list(s, ",", confs);
2230 for (auto c : confs) {
2231 ssize_t pos = c.find("=");
2235 out[c.substr(0, pos)] = c.substr(pos + 1);
2240 int check_reshard_bucket_params(RGWRados *store,
2241 const string& bucket_name,
2242 const string& tenant,
2243 const string& bucket_id,
2244 bool num_shards_specified,
2246 int yes_i_really_mean_it,
2248 RGWBucketInfo& bucket_info,
2249 map<string, bufferlist>& attrs)
2251 if (bucket_name.empty()) {
2252 cerr << "ERROR: bucket not specified" << std::endl;
2256 if (!num_shards_specified) {
2257 cerr << "ERROR: --num-shards not specified" << std::endl;
2261 if (num_shards > (int)store->get_max_bucket_shards()) {
2262 cerr << "ERROR: num_shards too high, max value: " << store->get_max_bucket_shards() << std::endl;
2266 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket, &attrs);
2268 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
2272 int num_source_shards = (bucket_info.num_shards > 0 ? bucket_info.num_shards : 1);
2274 if (num_shards <= num_source_shards && !yes_i_really_mean_it) {
2275 cerr << "num shards is less or equal to current shards count" << std::endl
2276 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
2282 int create_new_bucket_instance(RGWRados *store,
2284 const RGWBucketInfo& bucket_info,
2285 map<string, bufferlist>& attrs,
2286 RGWBucketInfo& new_bucket_info)
2289 store->create_bucket_id(&new_bucket_info.bucket.bucket_id);
2290 new_bucket_info.bucket.oid.clear();
2292 new_bucket_info.num_shards = new_num_shards;
2293 new_bucket_info.objv_tracker.clear();
2295 int ret = store->init_bucket_index(new_bucket_info, new_bucket_info.num_shards);
2297 cerr << "ERROR: failed to init new bucket indexes: " << cpp_strerror(-ret) << std::endl;
2301 ret = store->put_bucket_instance_info(new_bucket_info, true, real_time(), &attrs);
2303 cerr << "ERROR: failed to store new bucket instance info: " << cpp_strerror(-ret) << std::endl;
2311 #ifdef BUILDING_FOR_EMBEDDED
2312 extern "C" int cephd_rgw_admin(int argc, const char **argv)
2314 int main(int argc, const char **argv)
2317 vector<const char*> args;
2318 argv_to_vec(argc, (const char **)argv, args);
2321 auto cct = global_init(NULL, args, CEPH_ENTITY_TYPE_CLIENT,
2322 CODE_ENVIRONMENT_UTILITY, 0);
2324 // for region -> zonegroup conversion (must happen before common_init_finish())
2325 if (!g_conf->rgw_region.empty() && g_conf->rgw_zonegroup.empty()) {
2326 g_conf->set_val_or_die("rgw_zonegroup", g_conf->rgw_region.c_str());
2329 common_init_finish(g_ceph_context);
2333 std::string access_key, secret_key, user_email, display_name;
2334 std::string bucket_name, pool_name, object;
2336 std::string date, subuser, access, format;
2337 std::string start_date, end_date;
2338 std::string key_type_str;
2339 std::string period_id, period_epoch, remote, url;
2340 std::string master_zonegroup, master_zone;
2341 std::string realm_name, realm_id, realm_new_name;
2342 std::string zone_name, zone_id, zone_new_name;
2343 std::string zonegroup_name, zonegroup_id, zonegroup_new_name;
2344 std::string api_name;
2345 std::string role_name, path, assume_role_doc, policy_name, perm_policy_doc, path_prefix;
2346 list<string> endpoints;
2348 int sync_from_all_specified = false;
2349 bool sync_from_all = false;
2350 list<string> sync_from;
2351 list<string> sync_from_rm;
2352 std::string master_url;
2354 int set_default = 0;
2355 bool is_master = false;
2356 bool is_master_set = false;
2358 bool read_only = false;
2359 int is_read_only_set = false;
2361 int staging = false;
2362 int key_type = KEY_TYPE_UNDEFINED;
2364 uint32_t perm_mask = 0;
2366 int opt_cmd = OPT_NO_CMD;
2368 int gen_access_key = 0;
2369 int gen_secret_key = 0;
2370 bool set_perm = false;
2371 bool set_temp_url_key = false;
2372 map<int, string> temp_url_keys;
2374 Formatter *formatter = NULL;
2375 int purge_data = false;
2376 int pretty_format = false;
2377 int show_log_entries = true;
2378 int show_log_sum = true;
2379 int skip_zero_entries = false; // log show
2380 int purge_keys = false;
2381 int yes_i_really_mean_it = false;
2382 int delete_child_objects = false;
2384 int remove_bad = false;
2385 int check_head_obj_locator = false;
2386 int max_buckets = -1;
2387 bool max_buckets_specified = false;
2388 map<string, bool> categories;
2390 int check_objects = false;
2391 RGWUserAdminOpState user_op;
2392 RGWBucketAdminOpState bucket_op;
2394 string metadata_key;
2395 RGWObjVersionTracker objv_tracker;
2397 string start_marker;
2399 int max_entries = -1;
2400 bool max_entries_specified = false;
2402 bool admin_specified = false;
2404 bool system_specified = false;
2406 bool specified_shard_id = false;
2408 bool specified_daemon_id = false;
2412 string replica_log_type_str;
2413 ReplicaLogType replica_log_type = ReplicaLog_Invalid;
2416 string object_version;
2417 string placement_id;
2419 list<string> tags_add;
2420 list<string> tags_rm;
2422 int64_t max_objects = -1;
2423 int64_t max_size = -1;
2424 bool have_max_objects = false;
2425 bool have_max_size = false;
2426 int include_all = false;
2428 int sync_stats = false;
2429 int bypass_gc = false;
2430 int warnings_only = false;
2431 int inconsistent_index = false;
2433 int verbose = false;
2435 int extra_info = false;
2437 uint64_t min_rewrite_size = 4 * 1024 * 1024;
2438 uint64_t max_rewrite_size = ULLONG_MAX;
2439 uint64_t min_rewrite_stripe_size = 0;
2441 BIIndexType bi_index_type = PlainIdx;
2445 bool num_shards_specified = false;
2446 int max_concurrent_ios = 32;
2447 uint64_t orphan_stale_secs = (24 * 3600);
2450 std::ostringstream errs;
2454 string source_zone_name;
2455 string source_zone; /* zone id */
2458 bool tier_type_specified = false;
2460 map<string, string, ltstr_nocase> tier_config_add;
2461 map<string, string, ltstr_nocase> tier_config_rm;
2463 boost::optional<string> index_pool;
2464 boost::optional<string> data_pool;
2465 boost::optional<string> data_extra_pool;
2466 RGWBucketIndexType placement_index_type = RGWBIType_Normal;
2467 bool index_type_specified = false;
2469 boost::optional<std::string> compression_type;
2471 for (std::vector<const char*>::iterator i = args.begin(); i != args.end(); ) {
2472 if (ceph_argparse_double_dash(args, i)) {
2474 } else if (ceph_argparse_flag(args, i, "-h", "--help", (char*)NULL)) {
2477 } else if (ceph_argparse_witharg(args, i, &val, "-i", "--uid", (char*)NULL)) {
2478 user_id.from_str(val);
2479 } else if (ceph_argparse_witharg(args, i, &val, "--tenant", (char*)NULL)) {
2481 } else if (ceph_argparse_witharg(args, i, &val, "--access-key", (char*)NULL)) {
2483 } else if (ceph_argparse_witharg(args, i, &val, "--subuser", (char*)NULL)) {
2485 } else if (ceph_argparse_witharg(args, i, &val, "--secret", "--secret-key", (char*)NULL)) {
2487 } else if (ceph_argparse_witharg(args, i, &val, "-e", "--email", (char*)NULL)) {
2489 user_op.user_email_specified=true;
2490 } else if (ceph_argparse_witharg(args, i, &val, "-n", "--display-name", (char*)NULL)) {
2492 } else if (ceph_argparse_witharg(args, i, &val, "-b", "--bucket", (char*)NULL)) {
2494 } else if (ceph_argparse_witharg(args, i, &val, "-p", "--pool", (char*)NULL)) {
2496 pool = rgw_pool(pool_name);
2497 } else if (ceph_argparse_witharg(args, i, &val, "-o", "--object", (char*)NULL)) {
2499 } else if (ceph_argparse_witharg(args, i, &val, "--object-version", (char*)NULL)) {
2500 object_version = val;
2501 } else if (ceph_argparse_witharg(args, i, &val, "--client-id", (char*)NULL)) {
2503 } else if (ceph_argparse_witharg(args, i, &val, "--op-id", (char*)NULL)) {
2505 } else if (ceph_argparse_witharg(args, i, &val, "--state", (char*)NULL)) {
2507 } else if (ceph_argparse_witharg(args, i, &val, "--op-mask", (char*)NULL)) {
2509 } else if (ceph_argparse_witharg(args, i, &val, "--key-type", (char*)NULL)) {
2511 if (key_type_str.compare("swift") == 0) {
2512 key_type = KEY_TYPE_SWIFT;
2513 } else if (key_type_str.compare("s3") == 0) {
2514 key_type = KEY_TYPE_S3;
2516 cerr << "bad key type: " << key_type_str << std::endl;
2520 } else if (ceph_argparse_witharg(args, i, &val, "--job-id", (char*)NULL)) {
2522 } else if (ceph_argparse_binary_flag(args, i, &gen_access_key, NULL, "--gen-access-key", (char*)NULL)) {
2524 } else if (ceph_argparse_binary_flag(args, i, &gen_secret_key, NULL, "--gen-secret", (char*)NULL)) {
2526 } else if (ceph_argparse_binary_flag(args, i, &show_log_entries, NULL, "--show_log_entries", (char*)NULL)) {
2528 } else if (ceph_argparse_binary_flag(args, i, &show_log_sum, NULL, "--show_log_sum", (char*)NULL)) {
2530 } else if (ceph_argparse_binary_flag(args, i, &skip_zero_entries, NULL, "--skip_zero_entries", (char*)NULL)) {
2532 } else if (ceph_argparse_binary_flag(args, i, &admin, NULL, "--admin", (char*)NULL)) {
2533 admin_specified = true;
2534 } else if (ceph_argparse_binary_flag(args, i, &system, NULL, "--system", (char*)NULL)) {
2535 system_specified = true;
2536 } else if (ceph_argparse_binary_flag(args, i, &verbose, NULL, "--verbose", (char*)NULL)) {
2538 } else if (ceph_argparse_binary_flag(args, i, &staging, NULL, "--staging", (char*)NULL)) {
2540 } else if (ceph_argparse_binary_flag(args, i, &commit, NULL, "--commit", (char*)NULL)) {
2542 } else if (ceph_argparse_witharg(args, i, &tmp, errs, "-a", "--auth-uid", (char*)NULL)) {
2543 if (!errs.str().empty()) {
2544 cerr << errs.str() << std::endl;
2547 } else if (ceph_argparse_witharg(args, i, &val, "--min-rewrite-size", (char*)NULL)) {
2548 min_rewrite_size = (uint64_t)atoll(val.c_str());
2549 } else if (ceph_argparse_witharg(args, i, &val, "--max-rewrite-size", (char*)NULL)) {
2550 max_rewrite_size = (uint64_t)atoll(val.c_str());
2551 } else if (ceph_argparse_witharg(args, i, &val, "--min-rewrite-stripe-size", (char*)NULL)) {
2552 min_rewrite_stripe_size = (uint64_t)atoll(val.c_str());
2553 } else if (ceph_argparse_witharg(args, i, &val, "--max-buckets", (char*)NULL)) {
2554 max_buckets = (int)strict_strtol(val.c_str(), 10, &err);
2556 cerr << "ERROR: failed to parse max buckets: " << err << std::endl;
2559 max_buckets_specified = true;
2560 } else if (ceph_argparse_witharg(args, i, &val, "--max-entries", (char*)NULL)) {
2561 max_entries = (int)strict_strtol(val.c_str(), 10, &err);
2562 max_entries_specified = true;
2564 cerr << "ERROR: failed to parse max entries: " << err << std::endl;
2567 } else if (ceph_argparse_witharg(args, i, &val, "--max-size", (char*)NULL)) {
2568 max_size = strict_si_cast<int64_t>(val.c_str(), &err);
2570 cerr << "ERROR: failed to parse max size: " << err << std::endl;
2573 have_max_size = true;
2574 } else if (ceph_argparse_witharg(args, i, &val, "--max-objects", (char*)NULL)) {
2575 max_objects = (int64_t)strict_strtoll(val.c_str(), 10, &err);
2577 cerr << "ERROR: failed to parse max objects: " << err << std::endl;
2580 have_max_objects = true;
2581 } else if (ceph_argparse_witharg(args, i, &val, "--date", "--time", (char*)NULL)) {
2583 if (end_date.empty())
2585 } else if (ceph_argparse_witharg(args, i, &val, "--start-date", "--start-time", (char*)NULL)) {
2587 } else if (ceph_argparse_witharg(args, i, &val, "--end-date", "--end-time", (char*)NULL)) {
2589 } else if (ceph_argparse_witharg(args, i, &val, "--num-shards", (char*)NULL)) {
2590 num_shards = (int)strict_strtol(val.c_str(), 10, &err);
2592 cerr << "ERROR: failed to parse num shards: " << err << std::endl;
2595 num_shards_specified = true;
2596 } else if (ceph_argparse_witharg(args, i, &val, "--max-concurrent-ios", (char*)NULL)) {
2597 max_concurrent_ios = (int)strict_strtol(val.c_str(), 10, &err);
2599 cerr << "ERROR: failed to parse max concurrent ios: " << err << std::endl;
2602 } else if (ceph_argparse_witharg(args, i, &val, "--orphan-stale-secs", (char*)NULL)) {
2603 orphan_stale_secs = (uint64_t)strict_strtoll(val.c_str(), 10, &err);
2605 cerr << "ERROR: failed to parse orphan stale secs: " << err << std::endl;
2608 } else if (ceph_argparse_witharg(args, i, &val, "--shard-id", (char*)NULL)) {
2609 shard_id = (int)strict_strtol(val.c_str(), 10, &err);
2611 cerr << "ERROR: failed to parse shard id: " << err << std::endl;
2614 specified_shard_id = true;
2615 } else if (ceph_argparse_witharg(args, i, &val, "--daemon-id", (char*)NULL)) {
2617 specified_daemon_id = true;
2618 } else if (ceph_argparse_witharg(args, i, &val, "--access", (char*)NULL)) {
2620 perm_mask = rgw_str_to_perm(access.c_str());
2622 } else if (ceph_argparse_witharg(args, i, &val, "--temp-url-key", (char*)NULL)) {
2623 temp_url_keys[0] = val;
2624 set_temp_url_key = true;
2625 } else if (ceph_argparse_witharg(args, i, &val, "--temp-url-key2", "--temp-url-key-2", (char*)NULL)) {
2626 temp_url_keys[1] = val;
2627 set_temp_url_key = true;
2628 } else if (ceph_argparse_witharg(args, i, &val, "--bucket-id", (char*)NULL)) {
2630 if (bucket_id.empty()) {
2631 cerr << "bad bucket-id" << std::endl;
2635 } else if (ceph_argparse_witharg(args, i, &val, "--format", (char*)NULL)) {
2637 } else if (ceph_argparse_witharg(args, i, &val, "--categories", (char*)NULL)) {
2638 string cat_str = val;
2639 list<string> cat_list;
2640 list<string>::iterator iter;
2641 get_str_list(cat_str, cat_list);
2642 for (iter = cat_list.begin(); iter != cat_list.end(); ++iter) {
2643 categories[*iter] = true;
2645 } else if (ceph_argparse_binary_flag(args, i, &delete_child_objects, NULL, "--purge-objects", (char*)NULL)) {
2647 } else if (ceph_argparse_binary_flag(args, i, &pretty_format, NULL, "--pretty-format", (char*)NULL)) {
2649 } else if (ceph_argparse_binary_flag(args, i, &purge_data, NULL, "--purge-data", (char*)NULL)) {
2650 delete_child_objects = purge_data;
2651 } else if (ceph_argparse_binary_flag(args, i, &purge_keys, NULL, "--purge-keys", (char*)NULL)) {
2653 } else if (ceph_argparse_binary_flag(args, i, &yes_i_really_mean_it, NULL, "--yes-i-really-mean-it", (char*)NULL)) {
2655 } else if (ceph_argparse_binary_flag(args, i, &fix, NULL, "--fix", (char*)NULL)) {
2657 } else if (ceph_argparse_binary_flag(args, i, &remove_bad, NULL, "--remove-bad", (char*)NULL)) {
2659 } else if (ceph_argparse_binary_flag(args, i, &check_head_obj_locator, NULL, "--check-head-obj-locator", (char*)NULL)) {
2661 } else if (ceph_argparse_binary_flag(args, i, &check_objects, NULL, "--check-objects", (char*)NULL)) {
2663 } else if (ceph_argparse_binary_flag(args, i, &sync_stats, NULL, "--sync-stats", (char*)NULL)) {
2665 } else if (ceph_argparse_binary_flag(args, i, &include_all, NULL, "--include-all", (char*)NULL)) {
2667 } else if (ceph_argparse_binary_flag(args, i, &extra_info, NULL, "--extra-info", (char*)NULL)) {
2669 } else if (ceph_argparse_binary_flag(args, i, &bypass_gc, NULL, "--bypass-gc", (char*)NULL)) {
2671 } else if (ceph_argparse_binary_flag(args, i, &warnings_only, NULL, "--warnings-only", (char*)NULL)) {
2673 } else if (ceph_argparse_binary_flag(args, i, &inconsistent_index, NULL, "--inconsistent-index", (char*)NULL)) {
2675 } else if (ceph_argparse_witharg(args, i, &val, "--caps", (char*)NULL)) {
2677 } else if (ceph_argparse_witharg(args, i, &val, "-i", "--infile", (char*)NULL)) {
2679 } else if (ceph_argparse_witharg(args, i, &val, "--metadata-key", (char*)NULL)) {
2681 } else if (ceph_argparse_witharg(args, i, &val, "--marker", (char*)NULL)) {
2683 } else if (ceph_argparse_witharg(args, i, &val, "--start-marker", (char*)NULL)) {
2685 } else if (ceph_argparse_witharg(args, i, &val, "--end-marker", (char*)NULL)) {
2687 } else if (ceph_argparse_witharg(args, i, &val, "--quota-scope", (char*)NULL)) {
2689 } else if (ceph_argparse_witharg(args, i, &val, "--replica-log-type", (char*)NULL)) {
2690 replica_log_type_str = val;
2691 replica_log_type = get_replicalog_type(replica_log_type_str);
2692 if (replica_log_type == ReplicaLog_Invalid) {
2693 cerr << "ERROR: invalid replica log type" << std::endl;
2696 } else if (ceph_argparse_witharg(args, i, &val, "--index-type", (char*)NULL)) {
2697 string index_type_str = val;
2698 bi_index_type = get_bi_index_type(index_type_str);
2699 if (bi_index_type == InvalidIdx) {
2700 cerr << "ERROR: invalid bucket index entry type" << std::endl;
2703 } else if (ceph_argparse_binary_flag(args, i, &is_master_int, NULL, "--master", (char*)NULL)) {
2704 is_master = (bool)is_master_int;
2705 is_master_set = true;
2706 } else if (ceph_argparse_binary_flag(args, i, &set_default, NULL, "--default", (char*)NULL)) {
2708 } else if (ceph_argparse_binary_flag(args, i, &read_only_int, NULL, "--read-only", (char*)NULL)) {
2709 read_only = (bool)read_only_int;
2710 is_read_only_set = true;
2711 } else if (ceph_argparse_witharg(args, i, &val, "--master-url", (char*)NULL)) {
2713 } else if (ceph_argparse_witharg(args, i, &val, "--master-zonegroup", (char*)NULL)) {
2714 master_zonegroup = val;
2715 } else if (ceph_argparse_witharg(args, i, &val, "--master-zone", (char*)NULL)) {
2717 } else if (ceph_argparse_witharg(args, i, &val, "--period", (char*)NULL)) {
2719 } else if (ceph_argparse_witharg(args, i, &val, "--epoch", (char*)NULL)) {
2721 } else if (ceph_argparse_witharg(args, i, &val, "--remote", (char*)NULL)) {
2723 } else if (ceph_argparse_witharg(args, i, &val, "--url", (char*)NULL)) {
2725 } else if (ceph_argparse_witharg(args, i, &val, "--realm-id", (char*)NULL)) {
2727 } else if (ceph_argparse_witharg(args, i, &val, "--realm-new-name", (char*)NULL)) {
2728 realm_new_name = val;
2729 } else if (ceph_argparse_witharg(args, i, &val, "--zonegroup-id", (char*)NULL)) {
2731 } else if (ceph_argparse_witharg(args, i, &val, "--zonegroup-new-name", (char*)NULL)) {
2732 zonegroup_new_name = val;
2733 } else if (ceph_argparse_witharg(args, i, &val, "--placement-id", (char*)NULL)) {
2735 } else if (ceph_argparse_witharg(args, i, &val, "--tags", (char*)NULL)) {
2736 get_str_list(val, tags);
2737 } else if (ceph_argparse_witharg(args, i, &val, "--tags-add", (char*)NULL)) {
2738 get_str_list(val, tags_add);
2739 } else if (ceph_argparse_witharg(args, i, &val, "--tags-rm", (char*)NULL)) {
2740 get_str_list(val, tags_rm);
2741 } else if (ceph_argparse_witharg(args, i, &val, "--api-name", (char*)NULL)) {
2743 } else if (ceph_argparse_witharg(args, i, &val, "--zone-id", (char*)NULL)) {
2745 } else if (ceph_argparse_witharg(args, i, &val, "--zone-new-name", (char*)NULL)) {
2746 zone_new_name = val;
2747 } else if (ceph_argparse_witharg(args, i, &val, "--endpoints", (char*)NULL)) {
2748 get_str_list(val, endpoints);
2749 } else if (ceph_argparse_witharg(args, i, &val, "--sync-from", (char*)NULL)) {
2750 get_str_list(val, sync_from);
2751 } else if (ceph_argparse_witharg(args, i, &val, "--sync-from-rm", (char*)NULL)) {
2752 get_str_list(val, sync_from_rm);
2753 } else if (ceph_argparse_binary_flag(args, i, &tmp_int, NULL, "--sync-from-all", (char*)NULL)) {
2754 sync_from_all = (bool)tmp_int;
2755 sync_from_all_specified = true;
2756 } else if (ceph_argparse_witharg(args, i, &val, "--source-zone", (char*)NULL)) {
2757 source_zone_name = val;
2758 } else if (ceph_argparse_witharg(args, i, &val, "--tier-type", (char*)NULL)) {
2760 tier_type_specified = true;
2761 } else if (ceph_argparse_witharg(args, i, &val, "--tier-config", (char*)NULL)) {
2762 parse_tier_config_param(val, tier_config_add);
2763 } else if (ceph_argparse_witharg(args, i, &val, "--tier-config-rm", (char*)NULL)) {
2764 parse_tier_config_param(val, tier_config_rm);
2765 } else if (ceph_argparse_witharg(args, i, &val, "--index-pool", (char*)NULL)) {
2767 } else if (ceph_argparse_witharg(args, i, &val, "--data-pool", (char*)NULL)) {
2769 } else if (ceph_argparse_witharg(args, i, &val, "--data-extra-pool", (char*)NULL)) {
2770 data_extra_pool = val;
2771 } else if (ceph_argparse_witharg(args, i, &val, "--placement-index-type", (char*)NULL)) {
2772 if (val == "normal") {
2773 placement_index_type = RGWBIType_Normal;
2774 } else if (val == "indexless") {
2775 placement_index_type = RGWBIType_Indexless;
2777 placement_index_type = (RGWBucketIndexType)strict_strtol(val.c_str(), 10, &err);
2779 cerr << "ERROR: failed to parse index type index: " << err << std::endl;
2783 index_type_specified = true;
2784 } else if (ceph_argparse_witharg(args, i, &val, "--compression", (char*)NULL)) {
2785 compression_type = val;
2786 } else if (ceph_argparse_witharg(args, i, &val, "--role-name", (char*)NULL)) {
2788 } else if (ceph_argparse_witharg(args, i, &val, "--path", (char*)NULL)) {
2790 } else if (ceph_argparse_witharg(args, i, &val, "--assume-role-policy-doc", (char*)NULL)) {
2791 assume_role_doc = val;
2792 } else if (ceph_argparse_witharg(args, i, &val, "--policy-name", (char*)NULL)) {
2794 } else if (ceph_argparse_witharg(args, i, &val, "--policy-doc", (char*)NULL)) {
2795 perm_policy_doc = val;
2796 } else if (ceph_argparse_witharg(args, i, &val, "--path-prefix", (char*)NULL)) {
2798 } else if (strncmp(*i, "-", 1) == 0) {
2799 cerr << "ERROR: invalid flag " << *i << std::endl;
2811 const char *prev_cmd = NULL;
2812 const char *prev_prev_cmd = NULL;
2813 std::vector<const char*>::iterator i ;
2814 for (i = args.begin(); i != args.end(); ++i) {
2815 opt_cmd = get_cmd(*i, prev_cmd, prev_prev_cmd, &need_more);
2817 cerr << "unrecognized arg " << *i << std::endl;
2825 prev_prev_cmd = prev_cmd;
2829 if (opt_cmd == OPT_NO_CMD) {
2834 /* some commands may have an optional extra param */
2835 if (i != args.end()) {
2837 case OPT_METADATA_GET:
2838 case OPT_METADATA_PUT:
2839 case OPT_METADATA_RM:
2840 case OPT_METADATA_LIST:
2848 if (tenant.empty()) {
2849 tenant = user_id.tenant;
2851 if (user_id.empty() && opt_cmd != OPT_ROLE_CREATE
2852 && opt_cmd != OPT_ROLE_DELETE
2853 && opt_cmd != OPT_ROLE_GET
2854 && opt_cmd != OPT_ROLE_MODIFY
2855 && opt_cmd != OPT_ROLE_LIST
2856 && opt_cmd != OPT_ROLE_POLICY_PUT
2857 && opt_cmd != OPT_ROLE_POLICY_LIST
2858 && opt_cmd != OPT_ROLE_POLICY_GET
2859 && opt_cmd != OPT_ROLE_POLICY_DELETE) {
2860 cerr << "ERROR: --tenant is set, but there's no user ID" << std::endl;
2863 user_id.tenant = tenant;
2865 /* check key parameter conflict */
2866 if ((!access_key.empty()) && gen_access_key) {
2867 cerr << "ERROR: key parameter conflict, --access-key & --gen-access-key" << std::endl;
2870 if ((!secret_key.empty()) && gen_secret_key) {
2871 cerr << "ERROR: key parameter conflict, --secret & --gen-secret" << std::endl;
2876 // default to pretty json
2877 if (format.empty()) {
2879 pretty_format = true;
2882 if (format == "xml")
2883 formatter = new XMLFormatter(pretty_format);
2884 else if (format == "json")
2885 formatter = new JSONFormatter(pretty_format);
2887 cerr << "unrecognized format: " << format << std::endl;
2892 realm_name = g_conf->rgw_realm;
2893 zone_name = g_conf->rgw_zone;
2894 zonegroup_name = g_conf->rgw_zonegroup;
2896 RGWStreamFlusher f(formatter, cout);
2898 // not a raw op if 'period update' needs to commit to master
2899 bool raw_period_update = opt_cmd == OPT_PERIOD_UPDATE && !commit;
2900 std::set<int> raw_storage_ops_list = {OPT_ZONEGROUP_ADD, OPT_ZONEGROUP_CREATE, OPT_ZONEGROUP_DELETE,
2901 OPT_ZONEGROUP_GET, OPT_ZONEGROUP_LIST,
2902 OPT_ZONEGROUP_SET, OPT_ZONEGROUP_DEFAULT,
2903 OPT_ZONEGROUP_RENAME, OPT_ZONEGROUP_MODIFY,
2904 OPT_ZONEGROUP_REMOVE,
2905 OPT_ZONEGROUP_PLACEMENT_ADD, OPT_ZONEGROUP_PLACEMENT_RM,
2906 OPT_ZONEGROUP_PLACEMENT_MODIFY, OPT_ZONEGROUP_PLACEMENT_LIST,
2907 OPT_ZONEGROUP_PLACEMENT_DEFAULT,
2908 OPT_ZONE_CREATE, OPT_ZONE_DELETE,
2909 OPT_ZONE_GET, OPT_ZONE_SET, OPT_ZONE_RENAME,
2910 OPT_ZONE_LIST, OPT_ZONE_MODIFY, OPT_ZONE_DEFAULT,
2911 OPT_ZONE_PLACEMENT_ADD, OPT_ZONE_PLACEMENT_RM,
2912 OPT_ZONE_PLACEMENT_MODIFY, OPT_ZONE_PLACEMENT_LIST,
2914 OPT_PERIOD_DELETE, OPT_PERIOD_GET,
2916 OPT_PERIOD_GET_CURRENT, OPT_PERIOD_LIST,
2917 OPT_GLOBAL_QUOTA_GET, OPT_GLOBAL_QUOTA_SET,
2918 OPT_GLOBAL_QUOTA_ENABLE, OPT_GLOBAL_QUOTA_DISABLE,
2919 OPT_REALM_DELETE, OPT_REALM_GET, OPT_REALM_LIST,
2920 OPT_REALM_LIST_PERIODS,
2921 OPT_REALM_GET_DEFAULT, OPT_REALM_REMOVE,
2922 OPT_REALM_RENAME, OPT_REALM_SET,
2923 OPT_REALM_DEFAULT, OPT_REALM_PULL};
2926 bool raw_storage_op = (raw_storage_ops_list.find(opt_cmd) != raw_storage_ops_list.end() ||
2929 if (raw_storage_op) {
2930 store = RGWStoreManager::get_raw_storage(g_ceph_context);
2932 store = RGWStoreManager::get_storage(g_ceph_context, false, false, false, false, false);
2935 cerr << "couldn't init storage provider" << std::endl;
2939 if (!source_zone_name.empty()) {
2940 if (!store->find_zone_id_by_name(source_zone_name, &source_zone)) {
2941 cerr << "WARNING: cannot find source zone id for name=" << source_zone_name << std::endl;
2942 source_zone = source_zone_name;
2946 rgw_user_init(store);
2947 rgw_bucket_init(store->meta_mgr);
2949 StoreDestructor store_destructor(store);
2951 if (raw_storage_op) {
2953 case OPT_PERIOD_DELETE:
2955 if (period_id.empty()) {
2956 cerr << "missing period id" << std::endl;
2959 RGWPeriod period(period_id);
2960 int ret = period.init(g_ceph_context, store);
2962 cerr << "period.init failed: " << cpp_strerror(-ret) << std::endl;
2965 ret = period.delete_obj();
2967 cerr << "ERROR: couldn't delete period: " << cpp_strerror(-ret) << std::endl;
2973 case OPT_PERIOD_GET:
2976 if (!period_epoch.empty()) {
2977 epoch = atoi(period_epoch.c_str());
2980 RGWRealm realm(realm_id, realm_name);
2981 int ret = realm.init(g_ceph_context, store);
2983 cerr << "Error initializing realm " << cpp_strerror(-ret) << std::endl;
2986 realm_id = realm.get_id();
2987 realm_name = realm.get_name();
2988 period_id = RGWPeriod::get_staging_id(realm_id);
2991 RGWPeriod period(period_id, epoch);
2992 int ret = period.init(g_ceph_context, store, realm_id, realm_name);
2994 cerr << "period init failed: " << cpp_strerror(-ret) << std::endl;
2997 encode_json("period", period, formatter);
2998 formatter->flush(cout);
3002 case OPT_PERIOD_GET_CURRENT:
3004 int ret = read_current_period_id(store, realm_id, realm_name, &period_id);
3008 formatter->open_object_section("period_get_current");
3009 encode_json("current_period", period_id, formatter);
3010 formatter->close_section();
3011 formatter->flush(cout);
3014 case OPT_PERIOD_LIST:
3016 list<string> periods;
3017 int ret = store->list_periods(periods);
3019 cerr << "failed to list periods: " << cpp_strerror(-ret) << std::endl;
3022 formatter->open_object_section("periods_list");
3023 encode_json("periods", periods, formatter);
3024 formatter->close_section();
3025 formatter->flush(cout);
3029 case OPT_PERIOD_UPDATE:
3031 int ret = update_period(realm_id, realm_name, period_id, period_epoch,
3032 commit, remote, url, access_key, secret_key,
3033 formatter, yes_i_really_mean_it);
3039 case OPT_PERIOD_PULL:
3041 boost::optional<RGWRESTConn> conn;
3042 RGWRESTConn *remote_conn = nullptr;
3044 // load current period for endpoints
3045 RGWRealm realm(realm_id, realm_name);
3046 int ret = realm.init(g_ceph_context, store);
3048 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
3051 RGWPeriod current_period(realm.get_current_period());
3052 ret = current_period.init(g_ceph_context, store);
3054 cerr << "failed to init current period: " << cpp_strerror(-ret) << std::endl;
3057 if (remote.empty()) {
3058 // use realm master zone as remote
3059 remote = current_period.get_master_zone();
3061 conn = get_remote_conn(store, current_period.get_map(), remote);
3063 cerr << "failed to find a zone or zonegroup for remote "
3064 << remote << std::endl;
3067 remote_conn = &*conn;
3071 int ret = do_period_pull(remote_conn, url, access_key, secret_key,
3072 realm_id, realm_name, period_id, period_epoch,
3075 cerr << "period pull failed: " << cpp_strerror(-ret) << std::endl;
3079 encode_json("period", period, formatter);
3080 formatter->flush(cout);
3084 case OPT_GLOBAL_QUOTA_GET:
3085 case OPT_GLOBAL_QUOTA_SET:
3086 case OPT_GLOBAL_QUOTA_ENABLE:
3087 case OPT_GLOBAL_QUOTA_DISABLE:
3089 if (realm_id.empty()) {
3090 RGWRealm realm(g_ceph_context, store);
3091 if (!realm_name.empty()) {
3092 // look up realm_id for the given realm_name
3093 int ret = realm.read_id(realm_name, realm_id);
3095 cerr << "ERROR: failed to read realm for " << realm_name
3096 << ": " << cpp_strerror(-ret) << std::endl;
3100 // use default realm_id when none is given
3101 int ret = realm.read_default_id(realm_id);
3102 if (ret < 0 && ret != -ENOENT) { // on ENOENT, use empty realm_id
3103 cerr << "ERROR: failed to read default realm: "
3104 << cpp_strerror(-ret) << std::endl;
3110 RGWPeriodConfig period_config;
3111 int ret = period_config.read(store, realm_id);
3112 if (ret < 0 && ret != -ENOENT) {
3113 cerr << "ERROR: failed to read period config: "
3114 << cpp_strerror(-ret) << std::endl;
3118 formatter->open_object_section("period_config");
3119 if (quota_scope == "bucket") {
3120 set_quota_info(period_config.bucket_quota, opt_cmd,
3121 max_size, max_objects,
3122 have_max_size, have_max_objects);
3123 encode_json("bucket quota", period_config.bucket_quota, formatter);
3124 } else if (quota_scope == "user") {
3125 set_quota_info(period_config.user_quota, opt_cmd,
3126 max_size, max_objects,
3127 have_max_size, have_max_objects);
3128 encode_json("user quota", period_config.user_quota, formatter);
3129 } else if (quota_scope.empty() && opt_cmd == OPT_GLOBAL_QUOTA_GET) {
3130 // if no scope is given for GET, print both
3131 encode_json("bucket quota", period_config.bucket_quota, formatter);
3132 encode_json("user quota", period_config.user_quota, formatter);
3134 cerr << "ERROR: invalid quota scope specification. Please specify "
3135 "either --quota-scope=bucket, or --quota-scope=user" << std::endl;
3138 formatter->close_section();
3140 if (opt_cmd != OPT_GLOBAL_QUOTA_GET) {
3141 // write the modified period config
3142 ret = period_config.write(store, realm_id);
3144 cerr << "ERROR: failed to write period config: "
3145 << cpp_strerror(-ret) << std::endl;
3148 if (!realm_id.empty()) {
3149 cout << "Global quota changes saved. Use 'period update' to apply "
3150 "them to the staging period, and 'period commit' to commit the "
3151 "new period." << std::endl;
3153 cout << "Global quota changes saved. They will take effect as "
3154 "the gateways are restarted." << std::endl;
3158 formatter->flush(cout);
3162 case OPT_REALM_CREATE:
3164 if (realm_name.empty()) {
3165 cerr << "missing realm name" << std::endl;
3169 RGWRealm realm(realm_name, g_ceph_context, store);
3170 int ret = realm.create();
3172 cerr << "ERROR: couldn't create realm " << realm_name << ": " << cpp_strerror(-ret) << std::endl;
3177 ret = realm.set_as_default();
3179 cerr << "failed to set realm " << realm_name << " as default: " << cpp_strerror(-ret) << std::endl;
3183 encode_json("realm", realm, formatter);
3184 formatter->flush(cout);
3188 case OPT_REALM_DELETE:
3190 RGWRealm realm(realm_id, realm_name);
3191 if (realm_name.empty() && realm_id.empty()) {
3192 cerr << "missing realm name or id" << std::endl;
3195 int ret = realm.init(g_ceph_context, store);
3197 cerr << "realm.init failed: " << cpp_strerror(-ret) << std::endl;
3200 ret = realm.delete_obj();
3202 cerr << "ERROR: couldn't : " << cpp_strerror(-ret) << std::endl;
3210 RGWRealm realm(realm_id, realm_name);
3211 int ret = realm.init(g_ceph_context, store);
3213 if (ret == -ENOENT && realm_name.empty() && realm_id.empty()) {
3214 cerr << "missing realm name or id, or default realm not found" << std::endl;
3216 cerr << "realm.init failed: " << cpp_strerror(-ret) << std::endl;
3220 encode_json("realm", realm, formatter);
3221 formatter->flush(cout);
3225 case OPT_REALM_GET_DEFAULT:
3227 RGWRealm realm(g_ceph_context, store);
3229 int ret = realm.read_default_id(default_id);
3230 if (ret == -ENOENT) {
3231 cout << "No default realm is set" << std::endl;
3233 } else if (ret < 0) {
3234 cerr << "Error reading default realm:" << cpp_strerror(-ret) << std::endl;
3237 cout << "default realm: " << default_id << std::endl;
3240 case OPT_REALM_LIST:
3242 RGWRealm realm(g_ceph_context, store);
3244 int ret = realm.read_default_id(default_id);
3245 if (ret < 0 && ret != -ENOENT) {
3246 cerr << "could not determine default realm: " << cpp_strerror(-ret) << std::endl;
3248 list<string> realms;
3249 ret = store->list_realms(realms);
3251 cerr << "failed to list realms: " << cpp_strerror(-ret) << std::endl;
3254 formatter->open_object_section("realms_list");
3255 encode_json("default_info", default_id, formatter);
3256 encode_json("realms", realms, formatter);
3257 formatter->close_section();
3258 formatter->flush(cout);
3262 case OPT_REALM_LIST_PERIODS:
3264 int ret = read_current_period_id(store, realm_id, realm_name, &period_id);
3268 list<string> periods;
3269 ret = store->list_periods(period_id, periods);
3271 cerr << "list periods failed: " << cpp_strerror(-ret) << std::endl;
3274 formatter->open_object_section("realm_periods_list");
3275 encode_json("current_period", period_id, formatter);
3276 encode_json("periods", periods, formatter);
3277 formatter->close_section();
3278 formatter->flush(cout);
3283 case OPT_REALM_RENAME:
3285 RGWRealm realm(realm_id, realm_name);
3286 if (realm_new_name.empty()) {
3287 cerr << "missing realm new name" << std::endl;
3290 if (realm_name.empty() && realm_id.empty()) {
3291 cerr << "missing realm name or id" << std::endl;
3294 int ret = realm.init(g_ceph_context, store);
3296 cerr << "realm.init failed: " << cpp_strerror(-ret) << std::endl;
3299 ret = realm.rename(realm_new_name);
3301 cerr << "realm.rename failed: " << cpp_strerror(-ret) << std::endl;
3304 cout << "Realm name updated. Note that this change only applies to "
3305 "the current cluster, so this command must be run separately "
3306 "on each of the realm's other clusters." << std::endl;
3311 if (realm_id.empty() && realm_name.empty()) {
3312 cerr << "no realm name or id provided" << std::endl;
3315 RGWRealm realm(realm_id, realm_name);
3316 bool new_realm = false;
3317 int ret = realm.init(g_ceph_context, store);
3318 if (ret < 0 && ret != -ENOENT) {
3319 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
3321 } else if (ret == -ENOENT) {
3324 ret = read_decode_json(infile, realm);
3328 if (!realm_name.empty() && realm.get_name() != realm_name) {
3329 cerr << "mismatch between --rgw-realm " << realm_name << " and json input file name " <<
3330 realm.get_name() << std::endl;
3335 cout << "clearing period and epoch for new realm" << std::endl;
3336 realm.clear_current_period_and_epoch();
3337 ret = realm.create();
3339 cerr << "ERROR: couldn't create new realm: " << cpp_strerror(-ret) << std::endl;
3343 ret = realm.update();
3345 cerr << "ERROR: couldn't store realm info: " << cpp_strerror(-ret) << std::endl;
3351 ret = realm.set_as_default();
3353 cerr << "failed to set realm " << realm_name << " as default: " << cpp_strerror(-ret) << std::endl;
3356 encode_json("realm", realm, formatter);
3357 formatter->flush(cout);
3361 case OPT_REALM_DEFAULT:
3363 RGWRealm realm(realm_id, realm_name);
3364 int ret = realm.init(g_ceph_context, store);
3366 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
3369 ret = realm.set_as_default();
3371 cerr << "failed to set realm as default: " << cpp_strerror(-ret) << std::endl;
3376 case OPT_REALM_PULL:
3379 cerr << "A --url must be provided." << std::endl;
3383 req_info info(g_ceph_context, &env);
3384 info.method = "GET";
3385 info.request_uri = "/admin/realm";
3387 map<string, string> ¶ms = info.args.get_params();
3388 if (!realm_id.empty())
3389 params["id"] = realm_id;
3390 if (!realm_name.empty())
3391 params["name"] = realm_name;
3395 int ret = send_to_url(url, access_key, secret_key, info, bl, p);
3397 cerr << "request failed: " << cpp_strerror(-ret) << std::endl;
3398 if (ret == -EACCES) {
3399 cerr << "If the realm has been changed on the master zone, the "
3400 "master zone's gateway may need to be restarted to recognize "
3401 "this user." << std::endl;
3406 realm.init(g_ceph_context, store, false);
3408 decode_json_obj(realm, &p);
3409 } catch (JSONDecoder::err& e) {
3410 cerr << "failed to decode JSON response: " << e.message << std::endl;
3414 auto& current_period = realm.get_current_period();
3415 if (!current_period.empty()) {
3416 // pull the latest epoch of the realm's current period
3417 ret = do_period_pull(nullptr, url, access_key, secret_key,
3418 realm_id, realm_name, current_period, "",
3421 cerr << "could not fetch period " << current_period << std::endl;
3425 ret = realm.create(false);
3426 if (ret < 0 && ret != -EEXIST) {
3427 cerr << "Error storing realm " << realm.get_id() << ": "
3428 << cpp_strerror(ret) << std::endl;
3430 } else if (ret ==-EEXIST) {
3431 ret = realm.update();
3433 cerr << "Error storing realm " << realm.get_id() << ": "
3434 << cpp_strerror(ret) << std::endl;
3439 ret = realm.set_as_default();
3441 cerr << "failed to set realm " << realm_name << " as default: " << cpp_strerror(-ret) << std::endl;
3445 encode_json("realm", realm, formatter);
3446 formatter->flush(cout);
3451 case OPT_ZONEGROUP_ADD:
3453 if (zonegroup_id.empty() && zonegroup_name.empty()) {
3454 cerr << "no zonegroup name or id provided" << std::endl;
3458 RGWZoneGroup zonegroup(zonegroup_id,zonegroup_name);
3459 int ret = zonegroup.init(g_ceph_context, store);
3461 cerr << "failed to initialize zonegroup " << zonegroup_name << " id " << zonegroup_id << " :"
3462 << cpp_strerror(-ret) << std::endl;
3465 RGWZoneParams zone(zone_id, zone_name);
3466 ret = zone.init(g_ceph_context, store);
3468 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
3471 if (zone.realm_id != zonegroup.realm_id) {
3472 zone.realm_id = zonegroup.realm_id;
3473 ret = zone.update();
3475 cerr << "failed to save zone info: " << cpp_strerror(-ret) << std::endl;
3480 string *ptier_type = (tier_type_specified ? &tier_type : nullptr);
3481 zone.tier_config = tier_config_add;
3483 bool *psync_from_all = (sync_from_all_specified ? &sync_from_all : nullptr);
3485 ret = zonegroup.add_zone(zone,
3486 (is_master_set ? &is_master : NULL),
3487 (is_read_only_set ? &read_only : NULL),
3488 endpoints, ptier_type,
3489 psync_from_all, sync_from, sync_from_rm);
3491 cerr << "failed to add zone " << zone_name << " to zonegroup " << zonegroup.get_name() << ": "
3492 << cpp_strerror(-ret) << std::endl;
3496 encode_json("zonegroup", zonegroup, formatter);
3497 formatter->flush(cout);
3500 case OPT_ZONEGROUP_CREATE:
3502 if (zonegroup_name.empty()) {
3503 cerr << "Missing zonegroup name" << std::endl;
3506 RGWRealm realm(realm_id, realm_name);
3507 int ret = realm.init(g_ceph_context, store);
3509 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
3513 RGWZoneGroup zonegroup(zonegroup_name, is_master, g_ceph_context, store, realm.get_id(), endpoints);
3514 zonegroup.api_name = (api_name.empty() ? zonegroup_name : api_name);
3515 ret = zonegroup.create();
3517 cerr << "failed to create zonegroup " << zonegroup_name << ": " << cpp_strerror(-ret) << std::endl;
3522 ret = zonegroup.set_as_default();
3524 cerr << "failed to set zonegroup " << zonegroup_name << " as default: " << cpp_strerror(-ret) << std::endl;
3528 encode_json("zonegroup", zonegroup, formatter);
3529 formatter->flush(cout);
3533 case OPT_ZONEGROUP_DEFAULT:
3535 if (zonegroup_id.empty() && zonegroup_name.empty()) {
3536 cerr << "no zonegroup name or id provided" << std::endl;
3540 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3541 int ret = zonegroup.init(g_ceph_context, store);
3543 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3547 ret = zonegroup.set_as_default();
3549 cerr << "failed to set zonegroup as default: " << cpp_strerror(-ret) << std::endl;
3554 case OPT_ZONEGROUP_DELETE:
3556 if (zonegroup_id.empty() && zonegroup_name.empty()) {
3557 cerr << "no zonegroup name or id provided" << std::endl;
3560 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3561 int ret = zonegroup.init(g_ceph_context, store);
3563 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3566 ret = zonegroup.delete_obj();
3568 cerr << "ERROR: couldn't delete zonegroup: " << cpp_strerror(-ret) << std::endl;
3573 case OPT_ZONEGROUP_GET:
3575 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3576 int ret = zonegroup.init(g_ceph_context, store);
3578 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3582 encode_json("zonegroup", zonegroup, formatter);
3583 formatter->flush(cout);
3587 case OPT_ZONEGROUP_LIST:
3589 RGWZoneGroup zonegroup;
3590 int ret = zonegroup.init(g_ceph_context, store, false);
3592 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3596 list<string> zonegroups;
3597 ret = store->list_zonegroups(zonegroups);
3599 cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl;
3602 string default_zonegroup;
3603 ret = zonegroup.read_default_id(default_zonegroup);
3604 if (ret < 0 && ret != -ENOENT) {
3605 cerr << "could not determine default zonegroup: " << cpp_strerror(-ret) << std::endl;
3607 formatter->open_object_section("zonegroups_list");
3608 encode_json("default_info", default_zonegroup, formatter);
3609 encode_json("zonegroups", zonegroups, formatter);
3610 formatter->close_section();
3611 formatter->flush(cout);
3615 case OPT_ZONEGROUP_MODIFY:
3617 RGWRealm realm(realm_id, realm_name);
3618 int ret = realm.init(g_ceph_context, store);
3620 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
3624 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3625 ret = zonegroup.init(g_ceph_context, store);
3627 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3631 bool need_update = false;
3633 if (!master_zone.empty()) {
3634 zonegroup.master_zone = master_zone;
3638 if (is_master_set) {
3639 zonegroup.update_master(is_master);
3643 if (!endpoints.empty()) {
3644 zonegroup.endpoints = endpoints;
3648 if (!api_name.empty()) {
3649 zonegroup.api_name = api_name;
3653 if (!realm_id.empty()) {
3654 zonegroup.realm_id = realm_id;
3656 } else if (!realm_name.empty()) {
3657 // get realm id from name
3658 RGWRealm realm{g_ceph_context, store};
3659 ret = realm.read_id(realm_name, zonegroup.realm_id);
3661 cerr << "failed to find realm by name " << realm_name << std::endl;
3668 ret = zonegroup.update();
3670 cerr << "failed to update zonegroup: " << cpp_strerror(-ret) << std::endl;
3676 ret = zonegroup.set_as_default();
3678 cerr << "failed to set zonegroup " << zonegroup_name << " as default: " << cpp_strerror(-ret) << std::endl;
3682 encode_json("zonegroup", zonegroup, formatter);
3683 formatter->flush(cout);
3686 case OPT_ZONEGROUP_SET:
3688 RGWRealm realm(realm_id, realm_name);
3689 int ret = realm.init(g_ceph_context, store);
3691 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
3695 RGWZoneGroup zonegroup;
3696 ret = zonegroup.init(g_ceph_context, store, false);
3698 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3701 ret = read_decode_json(infile, zonegroup);
3705 if (zonegroup.realm_id.empty()) {
3706 zonegroup.realm_id = realm.get_id();
3708 ret = zonegroup.create();
3709 if (ret < 0 && ret != -EEXIST) {
3710 cerr << "ERROR: couldn't create zonegroup info: " << cpp_strerror(-ret) << std::endl;
3712 } else if (ret == -EEXIST) {
3713 ret = zonegroup.update();
3715 cerr << "ERROR: couldn't store zonegroup info: " << cpp_strerror(-ret) << std::endl;
3721 ret = zonegroup.set_as_default();
3723 cerr << "failed to set zonegroup " << zonegroup_name << " as default: " << cpp_strerror(-ret) << std::endl;
3727 encode_json("zonegroup", zonegroup, formatter);
3728 formatter->flush(cout);
3731 case OPT_ZONEGROUP_REMOVE:
3733 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3734 int ret = zonegroup.init(g_ceph_context, store);
3736 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3740 if (zone_id.empty()) {
3741 if (zone_name.empty()) {
3742 cerr << "no --zone-id or --rgw-zone name provided" << std::endl;
3745 // look up zone id by name
3746 for (auto& z : zonegroup.zones) {
3747 if (zone_name == z.second.name) {
3748 zone_id = z.second.id;
3752 if (zone_id.empty()) {
3753 cerr << "zone name " << zone_name << " not found in zonegroup "
3754 << zonegroup.get_name() << std::endl;
3759 ret = zonegroup.remove_zone(zone_id);
3761 cerr << "failed to remove zone: " << cpp_strerror(-ret) << std::endl;
3765 encode_json("zonegroup", zonegroup, formatter);
3766 formatter->flush(cout);
3769 case OPT_ZONEGROUP_RENAME:
3771 if (zonegroup_new_name.empty()) {
3772 cerr << " missing zonegroup new name" << std::endl;
3775 if (zonegroup_id.empty() && zonegroup_name.empty()) {
3776 cerr << "no zonegroup name or id provided" << std::endl;
3779 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3780 int ret = zonegroup.init(g_ceph_context, store);
3782 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3785 ret = zonegroup.rename(zonegroup_new_name);
3787 cerr << "failed to rename zonegroup: " << cpp_strerror(-ret) << std::endl;
3792 case OPT_ZONEGROUP_PLACEMENT_LIST:
3794 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3795 int ret = zonegroup.init(g_ceph_context, store);
3797 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3801 encode_json("placement_targets", zonegroup.placement_targets, formatter);
3802 formatter->flush(cout);
3806 case OPT_ZONEGROUP_PLACEMENT_ADD:
3807 case OPT_ZONEGROUP_PLACEMENT_MODIFY:
3808 case OPT_ZONEGROUP_PLACEMENT_RM:
3809 case OPT_ZONEGROUP_PLACEMENT_DEFAULT:
3811 if (placement_id.empty()) {
3812 cerr << "ERROR: --placement-id not specified" << std::endl;
3816 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3817 int ret = zonegroup.init(g_ceph_context, store);
3819 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
3823 if (opt_cmd == OPT_ZONEGROUP_PLACEMENT_ADD) {
3824 RGWZoneGroupPlacementTarget target;
3825 target.name = placement_id;
3826 for (auto& t : tags) {
3827 target.tags.insert(t);
3829 zonegroup.placement_targets[placement_id] = target;
3830 } else if (opt_cmd == OPT_ZONEGROUP_PLACEMENT_MODIFY) {
3831 RGWZoneGroupPlacementTarget& target = zonegroup.placement_targets[placement_id];
3832 if (!tags.empty()) {
3833 target.tags.clear();
3834 for (auto& t : tags) {
3835 target.tags.insert(t);
3838 target.name = placement_id;
3839 for (auto& t : tags_rm) {
3840 target.tags.erase(t);
3842 for (auto& t : tags_add) {
3843 target.tags.insert(t);
3845 } else if (opt_cmd == OPT_ZONEGROUP_PLACEMENT_RM) {
3846 zonegroup.placement_targets.erase(placement_id);
3847 } else if (opt_cmd == OPT_ZONEGROUP_PLACEMENT_DEFAULT) {
3848 if (!zonegroup.placement_targets.count(placement_id)) {
3849 cerr << "failed to find a zonegroup placement target named '"
3850 << placement_id << "'" << std::endl;
3853 zonegroup.default_placement = placement_id;
3856 zonegroup.post_process_params();
3857 ret = zonegroup.update();
3859 cerr << "failed to update zonegroup: " << cpp_strerror(-ret) << std::endl;
3863 encode_json("placement_targets", zonegroup.placement_targets, formatter);
3864 formatter->flush(cout);
3867 case OPT_ZONE_CREATE:
3869 if (zone_name.empty()) {
3870 cerr << "zone name not provided" << std::endl;
3874 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
3875 /* if the user didn't provide zonegroup info , create stand alone zone */
3876 if (!zonegroup_id.empty() || !zonegroup_name.empty()) {
3877 ret = zonegroup.init(g_ceph_context, store);
3879 cerr << "unable to initialize zonegroup " << zonegroup_name << ": " << cpp_strerror(-ret) << std::endl;
3882 if (realm_id.empty() && realm_name.empty()) {
3883 realm_id = zonegroup.realm_id;
3887 RGWZoneParams zone(zone_id, zone_name);
3888 ret = zone.init(g_ceph_context, store, false);
3890 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
3894 zone.system_key.id = access_key;
3895 zone.system_key.key = secret_key;
3896 zone.realm_id = realm_id;
3897 zone.tier_config = tier_config_add;
3899 ret = zone.create();
3901 cerr << "failed to create zone " << zone_name << ": " << cpp_strerror(-ret) << std::endl;
3905 if (!zonegroup_id.empty() || !zonegroup_name.empty()) {
3906 string *ptier_type = (tier_type_specified ? &tier_type : nullptr);
3907 bool *psync_from_all = (sync_from_all_specified ? &sync_from_all : nullptr);
3908 ret = zonegroup.add_zone(zone,
3909 (is_master_set ? &is_master : NULL),
3910 (is_read_only_set ? &read_only : NULL),
3914 sync_from, sync_from_rm);
3916 cerr << "failed to add zone " << zone_name << " to zonegroup " << zonegroup.get_name()
3917 << ": " << cpp_strerror(-ret) << std::endl;
3923 ret = zone.set_as_default();
3925 cerr << "failed to set zone " << zone_name << " as default: " << cpp_strerror(-ret) << std::endl;
3929 encode_json("zone", zone, formatter);
3930 formatter->flush(cout);
3934 case OPT_ZONE_DEFAULT:
3936 RGWZoneGroup zonegroup(zonegroup_id,zonegroup_name);
3937 int ret = zonegroup.init(g_ceph_context, store);
3939 cerr << "WARNING: failed to initialize zonegroup " << zonegroup_name << std::endl;
3941 if (zone_id.empty() && zone_name.empty()) {
3942 cerr << "no zone name or id provided" << std::endl;
3945 RGWZoneParams zone(zone_id, zone_name);
3946 ret = zone.init(g_ceph_context, store);
3948 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
3951 ret = zone.set_as_default();
3953 cerr << "failed to set zone as default: " << cpp_strerror(-ret) << std::endl;
3958 case OPT_ZONE_DELETE:
3960 if (zone_id.empty() && zone_name.empty()) {
3961 cerr << "no zone name or id provided" << std::endl;
3964 RGWZoneParams zone(zone_id, zone_name);
3965 int ret = zone.init(g_ceph_context, store);
3967 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
3971 list<string> zonegroups;
3972 ret = store->list_zonegroups(zonegroups);
3974 cerr << "failed to list zonegroups: " << cpp_strerror(-ret) << std::endl;
3978 for (list<string>::iterator iter = zonegroups.begin(); iter != zonegroups.end(); ++iter) {
3979 RGWZoneGroup zonegroup(string(), *iter);
3980 int ret = zonegroup.init(g_ceph_context, store);
3982 cerr << "WARNING: failed to initialize zonegroup " << zonegroup_name << std::endl;
3985 ret = zonegroup.remove_zone(zone.get_id());
3986 if (ret < 0 && ret != -ENOENT) {
3987 cerr << "failed to remove zone " << zone_name << " from zonegroup " << zonegroup.get_name() << ": "
3988 << cpp_strerror(-ret) << std::endl;
3992 ret = zone.delete_obj();
3994 cerr << "failed to delete zone " << zone_name << ": " << cpp_strerror(-ret) << std::endl;
4001 RGWZoneParams zone(zone_id, zone_name);
4002 int ret = zone.init(g_ceph_context, store);
4004 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
4007 encode_json("zone", zone, formatter);
4008 formatter->flush(cout);
4013 RGWZoneParams zone(zone_name);
4014 int ret = zone.init(g_ceph_context, store, false);
4020 if (ret < 0 && ret != -ENOENT) {
4021 cerr << "zone.read() returned ret=" << ret << std::endl;
4025 string orig_id = zone.get_id();
4027 ret = read_decode_json(infile, zone);
4032 if(zone.realm_id.empty()) {
4033 RGWRealm realm(realm_id, realm_name);
4034 int ret = realm.init(g_ceph_context, store);
4035 if (ret < 0 && ret != -ENOENT) {
4036 cerr << "failed to init realm: " << cpp_strerror(-ret) << std::endl;
4039 zone.realm_id = realm.get_id();
4042 if( !zone_name.empty() && !zone.get_name().empty() && zone.get_name() != zone_name) {
4043 cerr << "Error: zone name" << zone_name << " is different than the zone name " << zone.get_name() << " in the provided json " << std::endl;
4047 if (zone.get_name().empty()) {
4048 zone.set_name(zone_name);
4049 if (zone.get_name().empty()) {
4050 cerr << "no zone name specified" << std::endl;
4055 zone_name = zone.get_name();
4057 if (zone.get_id().empty()) {
4058 zone.set_id(orig_id);
4061 if (zone.get_id().empty()) {
4062 cerr << "no zone name id the json provided, assuming old format" << std::endl;
4063 if (zone_name.empty()) {
4064 cerr << "missing zone name" << std::endl;
4067 zone.set_name(zone_name);
4068 zone.set_id(zone_name);
4071 cerr << "zone id " << zone.get_id();
4072 ret = zone.fix_pool_names();
4074 cerr << "ERROR: couldn't fix zone: " << cpp_strerror(-ret) << std::endl;
4077 ret = zone.write(false);
4079 cerr << "ERROR: couldn't create zone: " << cpp_strerror(-ret) << std::endl;
4084 ret = zone.set_as_default();
4086 cerr << "failed to set zone " << zone_name << " as default: " << cpp_strerror(-ret) << std::endl;
4090 encode_json("zone", zone, formatter);
4091 formatter->flush(cout);
4097 int ret = store->list_zones(zones);
4099 cerr << "failed to list zones: " << cpp_strerror(-ret) << std::endl;
4104 ret = zone.init(g_ceph_context, store, false);
4106 cerr << "failed to init zone: " << cpp_strerror(-ret) << std::endl;
4109 string default_zone;
4110 ret = zone.read_default_id(default_zone);
4111 if (ret < 0 && ret != -ENOENT) {
4112 cerr << "could not determine default zone: " << cpp_strerror(-ret) << std::endl;
4114 formatter->open_object_section("zones_list");
4115 encode_json("default_info", default_zone, formatter);
4116 encode_json("zones", zones, formatter);
4117 formatter->close_section();
4118 formatter->flush(cout);
4122 case OPT_ZONE_MODIFY:
4124 RGWZoneParams zone(zone_id, zone_name);
4125 int ret = zone.init(g_ceph_context, store);
4127 cerr << "failed to init zone: " << cpp_strerror(-ret) << std::endl;
4131 bool need_zone_update = false;
4132 if (!access_key.empty()) {
4133 zone.system_key.id = access_key;
4134 need_zone_update = true;
4137 if (!secret_key.empty()) {
4138 zone.system_key.key = secret_key;
4139 need_zone_update = true;
4142 if (!realm_id.empty()) {
4143 zone.realm_id = realm_id;
4144 need_zone_update = true;
4145 } else if (!realm_name.empty()) {
4146 // get realm id from name
4147 RGWRealm realm{g_ceph_context, store};
4148 ret = realm.read_id(realm_name, zone.realm_id);
4150 cerr << "failed to find realm by name " << realm_name << std::endl;
4153 need_zone_update = true;
4156 for (auto add : tier_config_add) {
4157 zone.tier_config[add.first] = add.second;
4158 need_zone_update = true;
4161 for (auto rm : tier_config_rm) {
4162 zone.tier_config.erase(rm.first);
4163 need_zone_update = true;
4166 if (need_zone_update) {
4167 ret = zone.update();
4169 cerr << "failed to save zone info: " << cpp_strerror(-ret) << std::endl;
4174 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
4175 ret = zonegroup.init(g_ceph_context, store);
4177 cerr << "failed to init zonegroup: " << cpp_strerror(-ret) << std::endl;
4180 string *ptier_type = (tier_type_specified ? &tier_type : nullptr);
4182 bool *psync_from_all = (sync_from_all_specified ? &sync_from_all : nullptr);
4184 ret = zonegroup.add_zone(zone,
4185 (is_master_set ? &is_master : NULL),
4186 (is_read_only_set ? &read_only : NULL),
4187 endpoints, ptier_type,
4188 psync_from_all, sync_from, sync_from_rm);
4190 cerr << "failed to update zonegroup: " << cpp_strerror(-ret) << std::endl;
4194 ret = zonegroup.update();
4196 cerr << "failed to update zonegroup: " << cpp_strerror(-ret) << std::endl;
4201 ret = zone.set_as_default();
4203 cerr << "failed to set zone " << zone_name << " as default: " << cpp_strerror(-ret) << std::endl;
4207 encode_json("zone", zone, formatter);
4208 formatter->flush(cout);
4211 case OPT_ZONE_RENAME:
4213 if (zone_new_name.empty()) {
4214 cerr << " missing zone new name" << std::endl;
4217 if (zone_id.empty() && zone_name.empty()) {
4218 cerr << "no zonegroup name or id provided" << std::endl;
4221 RGWZoneParams zone(zone_id,zone_name);
4222 int ret = zone.init(g_ceph_context, store);
4224 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
4227 ret = zone.rename(zone_new_name);
4229 cerr << "failed to rename zone " << zone_name << " to " << zone_new_name << ": " << cpp_strerror(-ret)
4233 RGWZoneGroup zonegroup(zonegroup_id, zonegroup_name);
4234 ret = zonegroup.init(g_ceph_context, store);
4236 cerr << "WARNING: failed to initialize zonegroup " << zonegroup_name << std::endl;
4238 ret = zonegroup.rename_zone(zone);
4239 if (ret < 0 && ret ) {
4240 cerr << "Error in zonegroup rename for " << zone_name << ": " << cpp_strerror(-ret) << std::endl;
4246 case OPT_ZONE_PLACEMENT_ADD:
4247 case OPT_ZONE_PLACEMENT_MODIFY:
4248 case OPT_ZONE_PLACEMENT_RM:
4250 if (placement_id.empty()) {
4251 cerr << "ERROR: --placement-id not specified" << std::endl;
4254 // validate compression type
4255 if (compression_type && *compression_type != "random"
4256 && !Compressor::get_comp_alg_type(*compression_type)) {
4257 std::cerr << "Unrecognized compression type" << std::endl;
4261 RGWZoneParams zone(zone_id, zone_name);
4262 int ret = zone.init(g_ceph_context, store);
4264 cerr << "failed to init zone: " << cpp_strerror(-ret) << std::endl;
4268 if (opt_cmd == OPT_ZONE_PLACEMENT_ADD) {
4269 // pool names are required
4270 if (!index_pool || index_pool->empty() ||
4271 !data_pool || data_pool->empty()) {
4272 cerr << "ERROR: need to specify both --index-pool and --data-pool" << std::endl;
4276 RGWZonePlacementInfo& info = zone.placement_pools[placement_id];
4278 info.index_pool = *index_pool;
4279 info.data_pool = *data_pool;
4280 if (data_extra_pool) {
4281 info.data_extra_pool = *data_extra_pool;
4283 if (index_type_specified) {
4284 info.index_type = placement_index_type;
4286 if (compression_type) {
4287 info.compression_type = *compression_type;
4289 } else if (opt_cmd == OPT_ZONE_PLACEMENT_MODIFY) {
4290 auto p = zone.placement_pools.find(placement_id);
4291 if (p == zone.placement_pools.end()) {
4292 cerr << "ERROR: zone placement target '" << placement_id
4293 << "' not found" << std::endl;
4296 auto& info = p->second;
4297 if (index_pool && !index_pool->empty()) {
4298 info.index_pool = *index_pool;
4300 if (data_pool && !data_pool->empty()) {
4301 info.data_pool = *data_pool;
4303 if (data_extra_pool) {
4304 info.data_extra_pool = *data_extra_pool;
4306 if (index_type_specified) {
4307 info.index_type = placement_index_type;
4309 if (compression_type) {
4310 info.compression_type = *compression_type;
4312 } else if (opt_cmd == OPT_ZONE_PLACEMENT_RM) {
4313 zone.placement_pools.erase(placement_id);
4316 ret = zone.update();
4318 cerr << "failed to save zone info: " << cpp_strerror(-ret) << std::endl;
4322 encode_json("zone", zone, formatter);
4323 formatter->flush(cout);
4326 case OPT_ZONE_PLACEMENT_LIST:
4328 RGWZoneParams zone(zone_id, zone_name);
4329 int ret = zone.init(g_ceph_context, store);
4331 cerr << "unable to initialize zone: " << cpp_strerror(-ret) << std::endl;
4334 encode_json("placement_pools", zone.placement_pools, formatter);
4335 formatter->flush(cout);
4342 if (!user_id.empty()) {
4343 user_op.set_user_id(user_id);
4344 bucket_op.set_user_id(user_id);
4347 if (!display_name.empty())
4348 user_op.set_display_name(display_name);
4350 if (!user_email.empty())
4351 user_op.set_user_email(user_email);
4353 if (!access_key.empty())
4354 user_op.set_access_key(access_key);
4356 if (!secret_key.empty())
4357 user_op.set_secret_key(secret_key);
4359 if (!subuser.empty())
4360 user_op.set_subuser(subuser);
4363 user_op.set_caps(caps);
4365 user_op.set_purge_data(purge_data);
4368 user_op.set_purge_keys();
4371 user_op.set_generate_key();
4374 user_op.set_gen_secret(); // assume that a key pair should be created
4376 if (max_buckets_specified)
4377 user_op.set_max_buckets(max_buckets);
4379 if (admin_specified)
4380 user_op.set_admin(admin);
4382 if (system_specified)
4383 user_op.set_system(system);
4386 user_op.set_perm(perm_mask);
4388 if (set_temp_url_key) {
4389 map<int, string>::iterator iter = temp_url_keys.begin();
4390 for (; iter != temp_url_keys.end(); ++iter) {
4391 user_op.set_temp_url_key(iter->second, iter->first);
4395 if (!op_mask_str.empty()) {
4397 int ret = rgw_parse_op_type_list(op_mask_str, &op_mask);
4399 cerr << "failed to parse op_mask: " << cpp_strerror(-ret) << std::endl;
4403 user_op.set_op_mask(op_mask);
4406 if (key_type != KEY_TYPE_UNDEFINED)
4407 user_op.set_key_type(key_type);
4409 // set suspension operation parameters
4410 if (opt_cmd == OPT_USER_ENABLE)
4411 user_op.set_suspension(false);
4412 else if (opt_cmd == OPT_USER_SUSPEND)
4413 user_op.set_suspension(true);
4415 // RGWUser to use for user operations
4418 if (!user_id.empty() || !subuser.empty()) {
4419 ret = user.init(store, user_op);
4421 cerr << "user.init failed: " << cpp_strerror(-ret) << std::endl;
4426 /* populate bucket operation */
4427 bucket_op.set_bucket_name(bucket_name);
4428 bucket_op.set_object(object);
4429 bucket_op.set_check_objects(check_objects);
4430 bucket_op.set_delete_children(delete_child_objects);
4431 bucket_op.set_fix_index(fix);
4432 bucket_op.set_max_aio(max_concurrent_ios);
4434 // required to gather errors from operations
4435 std::string err_msg;
4437 bool output_user_info = true;
4442 case OPT_USER_CREATE:
4443 if (!user_op.has_existing_user()) {
4444 user_op.set_generate_key(); // generate a new key by default
4446 ret = user.add(user_op, &err_msg);
4448 cerr << "could not create user: " << err_msg << std::endl;
4449 if (ret == -ERR_INVALID_TENANT_NAME)
4454 if (!subuser.empty()) {
4455 ret = user.subusers.add(user_op, &err_msg);
4457 cerr << "could not create subuser: " << err_msg << std::endl;
4463 ret = user.remove(user_op, &err_msg);
4465 cerr << "could not remove user: " << err_msg << std::endl;
4469 output_user_info = false;
4471 case OPT_USER_ENABLE:
4472 case OPT_USER_SUSPEND:
4473 case OPT_USER_MODIFY:
4474 ret = user.modify(user_op, &err_msg);
4476 cerr << "could not modify user: " << err_msg << std::endl;
4481 case OPT_SUBUSER_CREATE:
4482 ret = user.subusers.add(user_op, &err_msg);
4484 cerr << "could not create subuser: " << err_msg << std::endl;
4489 case OPT_SUBUSER_MODIFY:
4490 ret = user.subusers.modify(user_op, &err_msg);
4492 cerr << "could not modify subuser: " << err_msg << std::endl;
4497 case OPT_SUBUSER_RM:
4498 ret = user.subusers.remove(user_op, &err_msg);
4500 cerr << "could not remove subuser: " << err_msg << std::endl;
4506 ret = user.caps.add(user_op, &err_msg);
4508 cerr << "could not add caps: " << err_msg << std::endl;
4514 ret = user.caps.remove(user_op, &err_msg);
4516 cerr << "could not remove caps: " << err_msg << std::endl;
4521 case OPT_KEY_CREATE:
4522 ret = user.keys.add(user_op, &err_msg);
4524 cerr << "could not create key: " << err_msg << std::endl;
4530 ret = user.keys.remove(user_op, &err_msg);
4532 cerr << "could not remove key: " << err_msg << std::endl;
4536 case OPT_PERIOD_PUSH:
4539 req_info info(g_ceph_context, &env);
4540 info.method = "POST";
4541 info.request_uri = "/admin/realm/period";
4543 map<string, string> ¶ms = info.args.get_params();
4544 if (!realm_id.empty())
4545 params["realm_id"] = realm_id;
4546 if (!realm_name.empty())
4547 params["realm_name"] = realm_name;
4548 if (!period_id.empty())
4549 params["period_id"] = period_id;
4550 if (!period_epoch.empty())
4551 params["epoch"] = period_epoch;
4554 RGWPeriod period(period_id);
4555 int ret = period.init(g_ceph_context, store);
4557 cerr << "period init failed: " << cpp_strerror(-ret) << std::endl;
4560 // json format into a bufferlist
4561 JSONFormatter jf(false);
4562 encode_json("period", period, &jf);
4567 ret = send_to_remote_or_url(nullptr, url, access_key, secret_key,
4570 cerr << "request failed: " << cpp_strerror(-ret) << std::endl;
4575 case OPT_PERIOD_UPDATE:
4577 int ret = update_period(realm_id, realm_name, period_id, period_epoch,
4578 commit, remote, url, access_key, secret_key,
4579 formatter, yes_i_really_mean_it);
4585 case OPT_PERIOD_COMMIT:
4587 // read realm and staging period
4588 RGWRealm realm(realm_id, realm_name);
4589 int ret = realm.init(g_ceph_context, store);
4591 cerr << "Error initializing realm: " << cpp_strerror(-ret) << std::endl;
4594 RGWPeriod period(RGWPeriod::get_staging_id(realm.get_id()), 1);
4595 ret = period.init(g_ceph_context, store, realm.get_id());
4597 cerr << "period init failed: " << cpp_strerror(-ret) << std::endl;
4600 ret = commit_period(realm, period, remote, url, access_key, secret_key,
4601 yes_i_really_mean_it);
4603 cerr << "failed to commit period: " << cpp_strerror(-ret) << std::endl;
4607 encode_json("period", period, formatter);
4608 formatter->flush(cout);
4612 case OPT_ROLE_CREATE:
4614 if (role_name.empty()) {
4615 cerr << "ERROR: role name is empty" << std::endl;
4619 if (assume_role_doc.empty()) {
4620 cerr << "ERROR: assume role policy document is empty" << std::endl;
4623 /* The following two calls will be replaced by read_decode_json or something
4624 similar when the code for AWS Policies is in places */
4626 int ret = read_input(assume_role_doc, bl);
4628 cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
4632 if (!p.parse(bl.c_str(), bl.length())) {
4633 cout << "ERROR: failed to parse JSON: " << assume_role_doc << std::endl;
4636 string trust_policy = bl.to_str();
4637 RGWRole role(g_ceph_context, store, role_name, path, trust_policy, tenant);
4638 ret = role.create(true);
4642 show_role_info(role, formatter);
4645 case OPT_ROLE_DELETE:
4647 if (role_name.empty()) {
4648 cerr << "ERROR: empty role name" << std::endl;
4651 RGWRole role(g_ceph_context, store, role_name, tenant);
4652 ret = role.delete_obj();
4656 cout << "role: " << role_name << " successfully deleted" << std::endl;
4661 if (role_name.empty()) {
4662 cerr << "ERROR: empty role name" << std::endl;
4665 RGWRole role(g_ceph_context, store, role_name, tenant);
4670 show_role_info(role, formatter);
4673 case OPT_ROLE_MODIFY:
4675 if (role_name.empty()) {
4676 cerr << "ERROR: role name is empty" << std::endl;
4680 if (assume_role_doc.empty()) {
4681 cerr << "ERROR: assume role policy document is empty" << std::endl;
4685 /* The following two calls will be replaced by read_decode_json or something
4686 similar when the code for AWS Policies is in place */
4688 int ret = read_input(assume_role_doc, bl);
4690 cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
4694 if (!p.parse(bl.c_str(), bl.length())) {
4695 cout << "ERROR: failed to parse JSON: " << assume_role_doc << std::endl;
4698 string trust_policy = bl.to_str();
4699 RGWRole role(g_ceph_context, store, role_name, tenant);
4704 role.update_trust_policy(trust_policy);
4705 ret = role.update();
4709 cout << "Assume role policy document updated successfully for role: " << role_name << std::endl;
4714 vector<RGWRole> result;
4715 ret = RGWRole::get_roles_by_path_prefix(store, g_ceph_context, path_prefix, tenant, result);
4719 show_roles_info(result, formatter);
4722 case OPT_ROLE_POLICY_PUT:
4724 if (role_name.empty()) {
4725 cerr << "role name is empty" << std::endl;
4729 if (policy_name.empty()) {
4730 cerr << "policy name is empty" << std::endl;
4734 if (perm_policy_doc.empty()) {
4735 cerr << "permission policy document is empty" << std::endl;
4739 /* The following two calls will be replaced by read_decode_json or something
4740 similar, when code for AWS Policies is in place.*/
4742 int ret = read_input(perm_policy_doc, bl);
4744 cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
4748 if (!p.parse(bl.c_str(), bl.length())) {
4749 cout << "ERROR: failed to parse JSON: " << std::endl;
4753 perm_policy = bl.c_str();
4755 RGWRole role(g_ceph_context, store, role_name, tenant);
4760 role.set_perm_policy(policy_name, perm_policy);
4761 ret = role.update();
4765 cout << "Permission policy attached successfully" << std::endl;
4768 case OPT_ROLE_POLICY_LIST:
4770 if (role_name.empty()) {
4771 cerr << "ERROR: Role name is empty" << std::endl;
4774 RGWRole role(g_ceph_context, store, role_name, tenant);
4779 std::vector<string> policy_names = role.get_role_policy_names();
4780 show_policy_names(policy_names, formatter);
4783 case OPT_ROLE_POLICY_GET:
4785 if (role_name.empty()) {
4786 cerr << "ERROR: role name is empty" << std::endl;
4790 if (policy_name.empty()) {
4791 cerr << "ERROR: policy name is empty" << std::endl;
4794 RGWRole role(g_ceph_context, store, role_name, tenant);
4795 int ret = role.get();
4800 ret = role.get_role_policy(policy_name, perm_policy);
4804 show_perm_policy(perm_policy, formatter);
4807 case OPT_ROLE_POLICY_DELETE:
4809 if (role_name.empty()) {
4810 cerr << "ERROR: role name is empty" << std::endl;
4814 if (policy_name.empty()) {
4815 cerr << "ERROR: policy name is empty" << std::endl;
4818 RGWRole role(g_ceph_context, store, role_name, tenant);
4823 ret = role.delete_policy(policy_name);
4827 ret = role.update();
4831 cout << "Policy: " << policy_name << " successfully deleted for role: "
4832 << role_name << std::endl;
4836 output_user_info = false;
4839 // output the result of a user operation
4840 if (output_user_info) {
4841 ret = user.info(info, &err_msg);
4843 cerr << "could not fetch user info: " << err_msg << std::endl;
4846 show_user_info(info, formatter);
4849 if (opt_cmd == OPT_POLICY) {
4850 if (format == "xml") {
4851 int ret = RGWBucketAdminOp::dump_s3_policy(store, bucket_op, cout);
4853 cerr << "ERROR: failed to get policy: " << cpp_strerror(-ret) << std::endl;
4857 int ret = RGWBucketAdminOp::get_policy(store, bucket_op, f);
4859 cerr << "ERROR: failed to get policy: " << cpp_strerror(-ret) << std::endl;
4865 if (opt_cmd == OPT_BUCKET_LIMIT_CHECK) {
4867 std::list<std::string> user_ids;
4868 metadata_key = "user";
4873 if (! user_id.empty()) {
4874 user_ids.push_back(user_id.id);
4876 RGWBucketAdminOp::limit_check(store, bucket_op, user_ids, f,
4879 /* list users in groups of max-keys, then perform user-bucket
4880 * limit-check on each group */
4881 ret = store->meta_mgr->list_keys_init(metadata_key, &handle);
4883 cerr << "ERROR: buckets limit check can't get user metadata_key: "
4884 << cpp_strerror(-ret) << std::endl;
4889 ret = store->meta_mgr->list_keys_next(handle, max, user_ids,
4891 if (ret < 0 && ret != -ENOENT) {
4892 cerr << "ERROR: buckets limit check lists_keys_next(): "
4893 << cpp_strerror(-ret) << std::endl;
4896 /* ok, do the limit checks for this group */
4898 RGWBucketAdminOp::limit_check(store, bucket_op, user_ids, f,
4904 } while (truncated);
4905 store->meta_mgr->list_keys_complete(handle);
4908 } /* OPT_BUCKET_LIMIT_CHECK */
4910 if (opt_cmd == OPT_BUCKETS_LIST) {
4911 if (bucket_name.empty()) {
4912 RGWBucketAdminOp::info(store, bucket_op, f);
4914 RGWBucketInfo bucket_info;
4915 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
4917 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
4920 formatter->open_array_section("entries");
4923 if (max_entries < 0)
4928 vector<rgw_bucket_dir_entry> result;
4929 map<string, bool> common_prefixes;
4932 RGWRados::Bucket target(store, bucket_info);
4933 RGWRados::Bucket::List list_op(&target);
4935 list_op.params.prefix = prefix;
4936 list_op.params.delim = delim;
4937 list_op.params.marker = rgw_obj_key(marker);
4938 list_op.params.ns = ns;
4939 list_op.params.enforce_ns = false;
4940 list_op.params.list_versions = true;
4943 ret = list_op.list_objects(max_entries - count, &result, &common_prefixes, &truncated);
4945 cerr << "ERROR: store->list_objects(): " << cpp_strerror(-ret) << std::endl;
4949 count += result.size();
4951 for (vector<rgw_bucket_dir_entry>::iterator iter = result.begin(); iter != result.end(); ++iter) {
4952 rgw_bucket_dir_entry& entry = *iter;
4953 encode_json("entry", entry, formatter);
4955 formatter->flush(cout);
4956 } while (truncated && count < max_entries);
4958 formatter->close_section();
4959 formatter->flush(cout);
4960 } /* have bucket_name */
4961 } /* OPT_BUCKETS_LIST */
4963 if (opt_cmd == OPT_BUCKET_STATS) {
4964 bucket_op.set_fetch_stats(true);
4966 int r = RGWBucketAdminOp::info(store, bucket_op, f);
4968 cerr << "failure: " << cpp_strerror(-r) << ": " << err << std::endl;
4973 if (opt_cmd == OPT_BUCKET_LINK) {
4974 bucket_op.set_bucket_id(bucket_id);
4976 int r = RGWBucketAdminOp::link(store, bucket_op, &err);
4978 cerr << "failure: " << cpp_strerror(-r) << ": " << err << std::endl;
4983 if (opt_cmd == OPT_BUCKET_UNLINK) {
4984 int r = RGWBucketAdminOp::unlink(store, bucket_op);
4986 cerr << "failure: " << cpp_strerror(-r) << std::endl;
4991 if (opt_cmd == OPT_LOG_LIST) {
4993 if (date.size() && date.size() != 10) {
4994 cerr << "bad date format for '" << date << "', expect YYYY-MM-DD" << std::endl;
4999 formatter->open_array_section("logs");
5001 int r = store->log_list_init(date, &h);
5006 cerr << "log list: error " << r << std::endl;
5011 int r = store->log_list_next(h, &name);
5015 cerr << "log list: error " << r << std::endl;
5018 formatter->dump_string("object", name);
5021 formatter->close_section();
5022 formatter->flush(cout);
5026 if (opt_cmd == OPT_LOG_SHOW || opt_cmd == OPT_LOG_RM) {
5027 if (object.empty() && (date.empty() || bucket_name.empty() || bucket_id.empty())) {
5028 cerr << "specify an object or a date, bucket and bucket-id" << std::endl;
5034 if (!object.empty()) {
5044 if (opt_cmd == OPT_LOG_SHOW) {
5047 int r = store->log_show_init(oid, &h);
5049 cerr << "error opening log " << oid << ": " << cpp_strerror(-r) << std::endl;
5054 formatter->open_object_section("log");
5056 struct rgw_log_entry entry;
5058 // peek at first entry to get bucket metadata
5059 r = store->log_show_next(h, &entry);
5061 cerr << "error reading log " << oid << ": " << cpp_strerror(-r) << std::endl;
5064 formatter->dump_string("bucket_id", entry.bucket_id);
5065 formatter->dump_string("bucket_owner", entry.bucket_owner.to_str());
5066 formatter->dump_string("bucket", entry.bucket);
5068 uint64_t agg_time = 0;
5069 uint64_t agg_bytes_sent = 0;
5070 uint64_t agg_bytes_received = 0;
5071 uint64_t total_entries = 0;
5073 if (show_log_entries)
5074 formatter->open_array_section("log_entries");
5077 uint64_t total_time = entry.total_time.sec() * 1000000LL + entry.total_time.usec();
5079 agg_time += total_time;
5080 agg_bytes_sent += entry.bytes_sent;
5081 agg_bytes_received += entry.bytes_received;
5084 if (skip_zero_entries && entry.bytes_sent == 0 &&
5085 entry.bytes_received == 0)
5088 if (show_log_entries) {
5090 rgw_format_ops_log_entry(entry, formatter);
5091 formatter->flush(cout);
5094 r = store->log_show_next(h, &entry);
5098 cerr << "error reading log " << oid << ": " << cpp_strerror(-r) << std::endl;
5101 if (show_log_entries)
5102 formatter->close_section();
5105 formatter->open_object_section("log_sum");
5106 formatter->dump_int("bytes_sent", agg_bytes_sent);
5107 formatter->dump_int("bytes_received", agg_bytes_received);
5108 formatter->dump_int("total_time", agg_time);
5109 formatter->dump_int("total_entries", total_entries);
5110 formatter->close_section();
5112 formatter->close_section();
5113 formatter->flush(cout);
5116 if (opt_cmd == OPT_LOG_RM) {
5117 int r = store->log_remove(oid);
5119 cerr << "error removing log " << oid << ": " << cpp_strerror(-r) << std::endl;
5125 if (opt_cmd == OPT_POOL_ADD) {
5126 if (pool_name.empty()) {
5127 cerr << "need to specify pool to add!" << std::endl;
5132 int ret = store->add_bucket_placement(pool);
5134 cerr << "failed to add bucket placement: " << cpp_strerror(-ret) << std::endl;
5137 if (opt_cmd == OPT_POOL_RM) {
5138 if (pool_name.empty()) {
5139 cerr << "need to specify pool to remove!" << std::endl;
5144 int ret = store->remove_bucket_placement(pool);
5146 cerr << "failed to remove bucket placement: " << cpp_strerror(-ret) << std::endl;
5149 if (opt_cmd == OPT_POOLS_LIST) {
5150 set<rgw_pool> pools;
5151 int ret = store->list_placement_set(pools);
5153 cerr << "could not list placement set: " << cpp_strerror(-ret) << std::endl;
5157 formatter->open_array_section("pools");
5158 for (auto siter = pools.begin(); siter != pools.end(); ++siter) {
5159 formatter->open_object_section("pool");
5160 formatter->dump_string("name", siter->to_str());
5161 formatter->close_section();
5163 formatter->close_section();
5164 formatter->flush(cout);
5168 if (opt_cmd == OPT_USAGE_SHOW) {
5169 uint64_t start_epoch = 0;
5170 uint64_t end_epoch = (uint64_t)-1;
5174 if (!start_date.empty()) {
5175 ret = utime_t::parse_date(start_date, &start_epoch, NULL);
5177 cerr << "ERROR: failed to parse start date" << std::endl;
5181 if (!end_date.empty()) {
5182 ret = utime_t::parse_date(end_date, &end_epoch, NULL);
5184 cerr << "ERROR: failed to parse end date" << std::endl;
5190 ret = RGWUsage::show(store, user_id, start_epoch, end_epoch,
5191 show_log_entries, show_log_sum, &categories,
5194 cerr << "ERROR: failed to show usage" << std::endl;
5199 if (opt_cmd == OPT_USAGE_TRIM) {
5200 if (user_id.empty() && !yes_i_really_mean_it) {
5201 cerr << "usage trim without user specified will remove *all* users data" << std::endl;
5202 cerr << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
5206 uint64_t start_epoch = 0;
5207 uint64_t end_epoch = (uint64_t)-1;
5210 if (!start_date.empty()) {
5211 ret = utime_t::parse_date(start_date, &start_epoch, NULL);
5213 cerr << "ERROR: failed to parse start date" << std::endl;
5218 if (!end_date.empty()) {
5219 ret = utime_t::parse_date(end_date, &end_epoch, NULL);
5221 cerr << "ERROR: failed to parse end date" << std::endl;
5226 ret = RGWUsage::trim(store, user_id, start_epoch, end_epoch);
5228 cerr << "ERROR: read_usage() returned ret=" << ret << std::endl;
5233 if (opt_cmd == OPT_OLH_GET || opt_cmd == OPT_OLH_READLOG) {
5234 if (bucket_name.empty()) {
5235 cerr << "ERROR: bucket not specified" << std::endl;
5238 if (object.empty()) {
5239 cerr << "ERROR: object not specified" << std::endl;
5244 if (opt_cmd == OPT_OLH_GET) {
5245 RGWBucketInfo bucket_info;
5246 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5248 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5252 rgw_obj obj(bucket, object);
5253 ret = store->get_olh(bucket_info, obj, &olh);
5255 cerr << "ERROR: failed reading olh: " << cpp_strerror(-ret) << std::endl;
5258 encode_json("olh", olh, formatter);
5259 formatter->flush(cout);
5262 if (opt_cmd == OPT_OLH_READLOG) {
5263 RGWBucketInfo bucket_info;
5264 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5266 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5269 map<uint64_t, vector<rgw_bucket_olh_log_entry> > log;
5272 RGWObjectCtx rctx(store);
5273 rgw_obj obj(bucket, object);
5277 ret = store->get_obj_state(&rctx, bucket_info, obj, &state, false); /* don't follow olh */
5282 ret = store->bucket_index_read_olh_log(bucket_info, *state, obj, 0, &log, &is_truncated);
5284 cerr << "ERROR: failed reading olh: " << cpp_strerror(-ret) << std::endl;
5287 formatter->open_object_section("result");
5288 encode_json("is_truncated", is_truncated, formatter);
5289 encode_json("log", log, formatter);
5290 formatter->close_section();
5291 formatter->flush(cout);
5294 if (opt_cmd == OPT_BI_GET) {
5295 RGWBucketInfo bucket_info;
5296 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5298 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5301 rgw_obj obj(bucket, object);
5302 if (!object_version.empty()) {
5303 obj.key.set_instance(object_version);
5306 rgw_cls_bi_entry entry;
5308 ret = store->bi_get(bucket, obj, bi_index_type, &entry);
5310 cerr << "ERROR: bi_get(): " << cpp_strerror(-ret) << std::endl;
5314 encode_json("entry", entry, formatter);
5315 formatter->flush(cout);
5318 if (opt_cmd == OPT_BI_PUT) {
5319 RGWBucketInfo bucket_info;
5320 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5322 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5326 rgw_cls_bi_entry entry;
5327 cls_rgw_obj_key key;
5328 ret = read_decode_json(infile, entry, &key);
5333 rgw_obj obj(bucket, key);
5335 ret = store->bi_put(bucket, obj, entry);
5337 cerr << "ERROR: bi_put(): " << cpp_strerror(-ret) << std::endl;
5342 if (opt_cmd == OPT_BI_LIST) {
5343 if (bucket_name.empty()) {
5344 cerr << "ERROR: bucket name not specified" << std::endl;
5347 RGWBucketInfo bucket_info;
5348 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5350 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5354 list<rgw_cls_bi_entry> entries;
5356 if (max_entries < 0) {
5360 int max_shards = (bucket_info.num_shards > 0 ? bucket_info.num_shards : 1);
5362 formatter->open_array_section("entries");
5364 for (int i = 0; i < max_shards; i++) {
5365 RGWRados::BucketShard bs(store);
5366 int shard_id = (bucket_info.num_shards > 0 ? i : -1);
5367 int ret = bs.init(bucket, shard_id);
5371 cerr << "ERROR: bs.init(bucket=" << bucket << ", shard=" << shard_id << "): " << cpp_strerror(-ret) << std::endl;
5377 ret = store->bi_list(bs, object, marker, max_entries, &entries, &is_truncated);
5379 cerr << "ERROR: bi_list(): " << cpp_strerror(-ret) << std::endl;
5383 list<rgw_cls_bi_entry>::iterator iter;
5384 for (iter = entries.begin(); iter != entries.end(); ++iter) {
5385 rgw_cls_bi_entry& entry = *iter;
5386 encode_json("entry", entry, formatter);
5389 formatter->flush(cout);
5390 } while (is_truncated);
5391 formatter->flush(cout);
5393 formatter->close_section();
5394 formatter->flush(cout);
5397 if (opt_cmd == OPT_BI_PURGE) {
5398 if (bucket_name.empty()) {
5399 cerr << "ERROR: bucket name not specified" << std::endl;
5402 RGWBucketInfo bucket_info;
5403 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5405 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5409 RGWBucketInfo cur_bucket_info;
5410 rgw_bucket cur_bucket;
5411 ret = init_bucket(tenant, bucket_name, string(), cur_bucket_info, cur_bucket);
5413 cerr << "ERROR: could not init current bucket info for bucket_name=" << bucket_name << ": " << cpp_strerror(-ret) << std::endl;
5417 if (cur_bucket_info.bucket.bucket_id == bucket_info.bucket.bucket_id && !yes_i_really_mean_it) {
5418 cerr << "specified bucket instance points to a current bucket instance" << std::endl;
5419 cerr << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
5423 int max_shards = (bucket_info.num_shards > 0 ? bucket_info.num_shards : 1);
5425 for (int i = 0; i < max_shards; i++) {
5426 RGWRados::BucketShard bs(store);
5427 int shard_id = (bucket_info.num_shards > 0 ? i : -1);
5428 int ret = bs.init(bucket, shard_id);
5430 cerr << "ERROR: bs.init(bucket=" << bucket << ", shard=" << shard_id << "): " << cpp_strerror(-ret) << std::endl;
5434 ret = store->bi_remove(bs);
5436 cerr << "ERROR: failed to remove bucket index object: " << cpp_strerror(-ret) << std::endl;
5442 if (opt_cmd == OPT_OBJECT_RM) {
5443 RGWBucketInfo bucket_info;
5444 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5446 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5449 rgw_obj_key key(object, object_version);
5450 ret = rgw_remove_object(store, bucket_info, bucket, key);
5453 cerr << "ERROR: object remove returned: " << cpp_strerror(-ret) << std::endl;
5458 if (opt_cmd == OPT_OBJECT_REWRITE) {
5459 if (bucket_name.empty()) {
5460 cerr << "ERROR: bucket not specified" << std::endl;
5463 if (object.empty()) {
5464 cerr << "ERROR: object not specified" << std::endl;
5468 RGWBucketInfo bucket_info;
5469 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5471 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5475 rgw_obj obj(bucket, object);
5476 obj.key.set_instance(object_version);
5477 bool need_rewrite = true;
5478 if (min_rewrite_stripe_size > 0) {
5479 ret = check_min_obj_stripe_size(store, bucket_info, obj, min_rewrite_stripe_size, &need_rewrite);
5481 ldout(store->ctx(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << ret << dendl;
5485 ret = store->rewrite_obj(bucket_info, obj);
5487 cerr << "ERROR: object rewrite returned: " << cpp_strerror(-ret) << std::endl;
5491 ldout(store->ctx(), 20) << "skipped object" << dendl;
5495 if (opt_cmd == OPT_OBJECTS_EXPIRE) {
5496 int ret = store->process_expire_objects();
5498 cerr << "ERROR: process_expire_objects() processing returned error: " << cpp_strerror(-ret) << std::endl;
5503 if (opt_cmd == OPT_BUCKET_REWRITE) {
5504 if (bucket_name.empty()) {
5505 cerr << "ERROR: bucket not specified" << std::endl;
5509 RGWBucketInfo bucket_info;
5510 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5512 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5516 uint64_t start_epoch = 0;
5517 uint64_t end_epoch = 0;
5519 if (!end_date.empty()) {
5520 int ret = utime_t::parse_date(end_date, &end_epoch, NULL);
5522 cerr << "ERROR: failed to parse end date" << std::endl;
5526 if (!start_date.empty()) {
5527 int ret = utime_t::parse_date(start_date, &start_epoch, NULL);
5529 cerr << "ERROR: failed to parse start date" << std::endl;
5534 bool is_truncated = true;
5536 rgw_obj_index_key marker;
5539 formatter->open_object_section("result");
5540 formatter->dump_string("bucket", bucket_name);
5541 formatter->open_array_section("objects");
5542 while (is_truncated) {
5543 map<string, rgw_bucket_dir_entry> result;
5544 int r = store->cls_bucket_list(bucket_info, RGW_NO_SHARD, marker, prefix, 1000, true,
5545 result, &is_truncated, &marker,
5546 bucket_object_check_filter);
5548 if (r < 0 && r != -ENOENT) {
5549 cerr << "ERROR: failed operation r=" << r << std::endl;
5555 map<string, rgw_bucket_dir_entry>::iterator iter;
5556 for (iter = result.begin(); iter != result.end(); ++iter) {
5557 rgw_obj_key key = iter->second.key;
5558 rgw_bucket_dir_entry& entry = iter->second;
5560 formatter->open_object_section("object");
5561 formatter->dump_string("name", key.name);
5562 formatter->dump_string("instance", key.instance);
5563 formatter->dump_int("size", entry.meta.size);
5564 utime_t ut(entry.meta.mtime);
5565 ut.gmtime(formatter->dump_stream("mtime"));
5567 if ((entry.meta.size < min_rewrite_size) ||
5568 (entry.meta.size > max_rewrite_size) ||
5569 (start_epoch > 0 && start_epoch > (uint64_t)ut.sec()) ||
5570 (end_epoch > 0 && end_epoch < (uint64_t)ut.sec())) {
5571 formatter->dump_string("status", "Skipped");
5573 rgw_obj obj(bucket, key);
5575 bool need_rewrite = true;
5576 if (min_rewrite_stripe_size > 0) {
5577 r = check_min_obj_stripe_size(store, bucket_info, obj, min_rewrite_stripe_size, &need_rewrite);
5579 ldout(store->ctx(), 0) << "WARNING: check_min_obj_stripe_size failed, r=" << r << dendl;
5582 if (!need_rewrite) {
5583 formatter->dump_string("status", "Skipped");
5585 r = store->rewrite_obj(bucket_info, obj);
5587 formatter->dump_string("status", "Success");
5589 formatter->dump_string("status", cpp_strerror(-r));
5593 formatter->dump_int("flags", entry.flags);
5595 formatter->close_section();
5596 formatter->flush(cout);
5599 formatter->close_section();
5600 formatter->close_section();
5601 formatter->flush(cout);
5604 if (opt_cmd == OPT_BUCKET_RESHARD) {
5606 RGWBucketInfo bucket_info;
5607 map<string, bufferlist> attrs;
5609 int ret = check_reshard_bucket_params(store,
5613 num_shards_specified,
5615 yes_i_really_mean_it,
5623 RGWBucketReshard br(store, bucket_info, attrs);
5625 #define DEFAULT_RESHARD_MAX_ENTRIES 1000
5626 if (max_entries < 1) {
5627 max_entries = DEFAULT_RESHARD_MAX_ENTRIES;
5630 return br.execute(num_shards, max_entries,
5631 verbose, &cout, formatter);
5634 if (opt_cmd == OPT_RESHARD_ADD) {
5636 RGWBucketInfo bucket_info;
5637 map<string, bufferlist> attrs;
5639 int ret = check_reshard_bucket_params(store,
5643 num_shards_specified,
5645 yes_i_really_mean_it,
5653 int num_source_shards = (bucket_info.num_shards > 0 ? bucket_info.num_shards : 1);
5655 RGWReshard reshard(store);
5656 cls_rgw_reshard_entry entry;
5657 entry.time = real_clock::now();
5658 entry.tenant = tenant;
5659 entry.bucket_name = bucket_name;
5660 entry.bucket_id = bucket_info.bucket.bucket_id;
5661 entry.old_num_shards = num_source_shards;
5662 entry.new_num_shards = num_shards;
5664 return reshard.add(entry);
5667 if (opt_cmd == OPT_RESHARD_LIST) {
5668 list<cls_rgw_reshard_entry> entries;
5671 if (max_entries < 0) {
5675 int num_logshards = store->ctx()->_conf->rgw_reshard_num_logs;
5677 RGWReshard reshard(store);
5679 formatter->open_array_section("reshard");
5680 for (int i = 0; i < num_logshards; i++) {
5681 bool is_truncated = true;
5685 ret = reshard.list(i, marker, max_entries, entries, &is_truncated);
5687 cerr << "Error listing resharding buckets: " << cpp_strerror(-ret) << std::endl;
5690 for (auto iter=entries.begin(); iter != entries.end(); ++iter) {
5691 cls_rgw_reshard_entry& entry = *iter;
5692 encode_json("entry", entry, formatter);
5693 entry.get_key(&marker);
5695 count += entries.size();
5696 formatter->flush(cout);
5697 } while (is_truncated && count < max_entries);
5699 if (count >= max_entries) {
5704 formatter->close_section();
5705 formatter->flush(cout);
5710 if (opt_cmd == OPT_RESHARD_STATUS) {
5711 if (bucket_name.empty()) {
5712 cerr << "ERROR: bucket not specified" << std::endl;
5717 RGWBucketInfo bucket_info;
5718 map<string, bufferlist> attrs;
5719 ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket, &attrs);
5721 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5725 RGWBucketReshard br(store, bucket_info, attrs);
5726 list<cls_rgw_bucket_instance_entry> status;
5727 int r = br.get_status(&status);
5729 cerr << "ERROR: could not get resharding status for bucket " << bucket_name << std::endl;
5733 encode_json("status", status, formatter);
5734 formatter->flush(cout);
5737 if (opt_cmd == OPT_RESHARD_PROCESS) {
5738 RGWReshard reshard(store, true, &cout);
5740 int ret = reshard.process_all_logshards();
5742 cerr << "ERROR: failed to process reshard logs, error=" << cpp_strerror(-ret) << std::endl;
5747 if (opt_cmd == OPT_RESHARD_CANCEL) {
5748 RGWReshard reshard(store);
5750 if (bucket_name.empty()) {
5751 cerr << "ERROR: bucket not specified" << std::endl;
5754 cls_rgw_reshard_entry entry;
5755 //entry.tenant = tenant;
5756 entry.bucket_name = bucket_name;
5757 //entry.bucket_id = bucket_id;
5758 int ret = reshard.get(entry);
5760 cerr << "Error in getting bucket " << bucket_name << ": " << cpp_strerror(-ret) << std::endl;
5764 /* TBD stop running resharding */
5766 ret =reshard.remove(entry);
5768 cerr << "Error removing bucket " << bucket_name << " for resharding queue: " << cpp_strerror(-ret) <<
5774 if (opt_cmd == OPT_OBJECT_UNLINK) {
5775 RGWBucketInfo bucket_info;
5776 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5778 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5781 list<rgw_obj_index_key> oid_list;
5782 rgw_obj_key key(object, object_version);
5783 rgw_obj_index_key index_key;
5784 key.get_index_key(&index_key);
5785 oid_list.push_back(index_key);
5786 ret = store->remove_objs_from_index(bucket_info, oid_list);
5788 cerr << "ERROR: remove_obj_from_index() returned error: " << cpp_strerror(-ret) << std::endl;
5793 if (opt_cmd == OPT_OBJECT_STAT) {
5794 RGWBucketInfo bucket_info;
5795 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
5797 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
5800 rgw_obj obj(bucket, object);
5801 obj.key.set_instance(object_version);
5804 map<string, bufferlist> attrs;
5805 RGWObjectCtx obj_ctx(store);
5806 RGWRados::Object op_target(store, bucket_info, obj_ctx, obj);
5807 RGWRados::Object::Read read_op(&op_target);
5809 read_op.params.attrs = &attrs;
5810 read_op.params.obj_size = &obj_size;
5812 ret = read_op.prepare();
5814 cerr << "ERROR: failed to stat object, returned error: " << cpp_strerror(-ret) << std::endl;
5817 formatter->open_object_section("object_metadata");
5818 formatter->dump_string("name", object);
5819 formatter->dump_unsigned("size", obj_size);
5821 map<string, bufferlist>::iterator iter;
5822 map<string, bufferlist> other_attrs;
5823 for (iter = attrs.begin(); iter != attrs.end(); ++iter) {
5824 bufferlist& bl = iter->second;
5825 bool handled = false;
5826 if (iter->first == RGW_ATTR_MANIFEST) {
5827 handled = decode_dump<RGWObjManifest>("manifest", bl, formatter);
5828 } else if (iter->first == RGW_ATTR_ACL) {
5829 handled = decode_dump<RGWAccessControlPolicy>("policy", bl, formatter);
5830 } else if (iter->first == RGW_ATTR_ID_TAG) {
5831 handled = dump_string("tag", bl, formatter);
5832 } else if (iter->first == RGW_ATTR_ETAG) {
5833 handled = dump_string("etag", bl, formatter);
5837 other_attrs[iter->first] = bl;
5840 formatter->open_object_section("attrs");
5841 for (iter = other_attrs.begin(); iter != other_attrs.end(); ++iter) {
5842 dump_string(iter->first.c_str(), iter->second, formatter);
5844 formatter->close_section();
5845 formatter->close_section();
5846 formatter->flush(cout);
5849 if (opt_cmd == OPT_BUCKET_CHECK) {
5850 if (check_head_obj_locator) {
5851 if (bucket_name.empty()) {
5852 cerr << "ERROR: need to specify bucket name" << std::endl;
5855 do_check_object_locator(tenant, bucket_name, fix, remove_bad, formatter);
5857 RGWBucketAdminOp::check_index(store, bucket_op, f);
5861 if (opt_cmd == OPT_BUCKET_RM) {
5862 if (inconsistent_index == false) {
5863 RGWBucketAdminOp::remove_bucket(store, bucket_op, bypass_gc, true);
5865 if (!yes_i_really_mean_it) {
5866 cerr << "using --inconsistent_index can corrupt the bucket index " << std::endl
5867 << "do you really mean it? (requires --yes-i-really-mean-it)" << std::endl;
5870 RGWBucketAdminOp::remove_bucket(store, bucket_op, bypass_gc, false);
5874 if (opt_cmd == OPT_GC_LIST) {
5877 formatter->open_array_section("entries");
5880 list<cls_rgw_gc_obj_info> result;
5881 int ret = store->list_gc_objs(&index, marker, 1000, !include_all, result, &truncated);
5883 cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl;
5888 list<cls_rgw_gc_obj_info>::iterator iter;
5889 for (iter = result.begin(); iter != result.end(); ++iter) {
5890 cls_rgw_gc_obj_info& info = *iter;
5891 formatter->open_object_section("chain_info");
5892 formatter->dump_string("tag", info.tag);
5893 formatter->dump_stream("time") << info.time;
5894 formatter->open_array_section("objs");
5895 list<cls_rgw_obj>::iterator liter;
5896 cls_rgw_obj_chain& chain = info.chain;
5897 for (liter = chain.objs.begin(); liter != chain.objs.end(); ++liter) {
5898 cls_rgw_obj& obj = *liter;
5899 encode_json("obj", obj, formatter);
5901 formatter->close_section(); // objs
5902 formatter->close_section(); // obj_chain
5903 formatter->flush(cout);
5905 } while (truncated);
5906 formatter->close_section();
5907 formatter->flush(cout);
5910 if (opt_cmd == OPT_GC_PROCESS) {
5911 int ret = store->process_gc();
5913 cerr << "ERROR: gc processing returned error: " << cpp_strerror(-ret) << std::endl;
5918 if (opt_cmd == OPT_LC_LIST) {
5919 formatter->open_array_section("lifecycle_list");
5920 map<string, int> bucket_lc_map;
5922 #define MAX_LC_LIST_ENTRIES 100
5923 if (max_entries < 0) {
5924 max_entries = MAX_LC_LIST_ENTRIES;
5927 int ret = store->list_lc_progress(marker, max_entries, &bucket_lc_map);
5929 cerr << "ERROR: failed to list objs: " << cpp_strerror(-ret) << std::endl;
5932 map<string, int>::iterator iter;
5933 for (iter = bucket_lc_map.begin(); iter != bucket_lc_map.end(); ++iter) {
5934 formatter->open_object_section("bucket_lc_info");
5935 formatter->dump_string("bucket", iter->first);
5936 string lc_status = LC_STATUS[iter->second];
5937 formatter->dump_string("status", lc_status);
5938 formatter->close_section(); // objs
5939 formatter->flush(cout);
5940 marker = iter->first;
5942 } while (!bucket_lc_map.empty());
5944 formatter->close_section(); //lifecycle list
5945 formatter->flush(cout);
5949 if (opt_cmd == OPT_LC_PROCESS) {
5950 int ret = store->process_lc();
5952 cerr << "ERROR: lc processing returned error: " << cpp_strerror(-ret) << std::endl;
5957 if (opt_cmd == OPT_ORPHANS_FIND) {
5958 RGWOrphanSearch search(store, max_concurrent_ios, orphan_stale_secs);
5960 if (job_id.empty()) {
5961 cerr << "ERROR: --job-id not specified" << std::endl;
5964 if (pool_name.empty()) {
5965 cerr << "ERROR: --pool not specified" << std::endl;
5969 RGWOrphanSearchInfo info;
5972 info.job_name = job_id;
5973 info.num_shards = num_shards;
5975 int ret = search.init(job_id, &info);
5977 cerr << "could not init search, ret=" << ret << std::endl;
5986 if (opt_cmd == OPT_ORPHANS_FINISH) {
5987 RGWOrphanSearch search(store, max_concurrent_ios, orphan_stale_secs);
5989 if (job_id.empty()) {
5990 cerr << "ERROR: --job-id not specified" << std::endl;
5993 int ret = search.init(job_id, NULL);
5995 if (ret == -ENOENT) {
5996 cerr << "job not found" << std::endl;
6000 ret = search.finish();
6006 if (opt_cmd == OPT_ORPHANS_LIST_JOBS){
6007 RGWOrphanStore orphan_store(store);
6008 int ret = orphan_store.init();
6010 cerr << "connection to cluster failed!" << std::endl;
6014 map <string,RGWOrphanSearchState> m;
6015 ret = orphan_store.list_jobs(m);
6017 cerr << "job list failed" << std::endl;
6020 formatter->open_array_section("entries");
6021 for (const auto &it: m){
6023 formatter->dump_string("job-id",it.first);
6025 encode_json("orphan_search_state", it.second, formatter);
6028 formatter->close_section();
6029 formatter->flush(cout);
6032 if (opt_cmd == OPT_USER_CHECK) {
6033 check_bad_user_bucket_mapping(store, user_id, fix);
6036 if (opt_cmd == OPT_USER_STATS) {
6038 if (!bucket_name.empty()) {
6039 int ret = rgw_bucket_sync_user_stats(store, tenant, bucket_name);
6041 cerr << "ERROR: could not sync bucket stats: " << cpp_strerror(-ret) << std::endl;
6045 int ret = rgw_user_sync_all_stats(store, user_id);
6047 cerr << "ERROR: failed to sync user stats: " << cpp_strerror(-ret) << std::endl;
6053 if (user_id.empty()) {
6054 cerr << "ERROR: uid not specified" << std::endl;
6057 cls_user_header header;
6058 string user_str = user_id.to_str();
6059 int ret = store->cls_user_get_header(user_str, &header);
6061 if (ret == -ENOENT) { /* in case of ENOENT */
6062 cerr << "User has not been initialized or user does not exist" << std::endl;
6064 cerr << "ERROR: can't read user: " << cpp_strerror(ret) << std::endl;
6069 encode_json("header", header, formatter);
6070 formatter->flush(cout);
6073 if (opt_cmd == OPT_METADATA_GET) {
6074 int ret = store->meta_mgr->get(metadata_key, formatter);
6076 cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl;
6080 formatter->flush(cout);
6083 if (opt_cmd == OPT_METADATA_PUT) {
6085 int ret = read_input(infile, bl);
6087 cerr << "ERROR: failed to read input: " << cpp_strerror(-ret) << std::endl;
6090 ret = store->meta_mgr->put(metadata_key, bl, RGWMetadataHandler::APPLY_ALWAYS);
6092 cerr << "ERROR: can't put key: " << cpp_strerror(-ret) << std::endl;
6097 if (opt_cmd == OPT_METADATA_RM) {
6098 int ret = store->meta_mgr->remove(metadata_key);
6100 cerr << "ERROR: can't remove key: " << cpp_strerror(-ret) << std::endl;
6105 if (opt_cmd == OPT_METADATA_LIST || opt_cmd == OPT_USER_LIST) {
6106 if (opt_cmd == OPT_USER_LIST) {
6107 metadata_key = "user";
6111 int ret = store->meta_mgr->list_keys_init(metadata_key, marker, &handle);
6113 cerr << "ERROR: can't get key: " << cpp_strerror(-ret) << std::endl;
6120 if (max_entries_specified) {
6121 formatter->open_object_section("result");
6123 formatter->open_array_section("keys");
6128 left = (max_entries_specified ? max_entries - count : max);
6129 ret = store->meta_mgr->list_keys_next(handle, left, keys, &truncated);
6130 if (ret < 0 && ret != -ENOENT) {
6131 cerr << "ERROR: lists_keys_next(): " << cpp_strerror(-ret) << std::endl;
6133 } if (ret != -ENOENT) {
6134 for (list<string>::iterator iter = keys.begin(); iter != keys.end(); ++iter) {
6135 formatter->dump_string("key", *iter);
6138 formatter->flush(cout);
6140 } while (truncated && left > 0);
6142 formatter->close_section();
6144 if (max_entries_specified) {
6145 encode_json("truncated", truncated, formatter);
6146 encode_json("count", count, formatter);
6148 encode_json("marker", store->meta_mgr->get_marker(handle), formatter);
6150 formatter->close_section();
6152 formatter->flush(cout);
6154 store->meta_mgr->list_keys_complete(handle);
6157 if (opt_cmd == OPT_MDLOG_LIST) {
6158 utime_t start_time, end_time;
6160 int ret = parse_date_str(start_date, start_time);
6164 ret = parse_date_str(end_date, end_time);
6168 int i = (specified_shard_id ? shard_id : 0);
6170 if (period_id.empty()) {
6171 int ret = read_current_period_id(store, realm_id, realm_name, &period_id);
6175 std::cerr << "No --period given, using current period="
6176 << period_id << std::endl;
6178 RGWMetadataLog *meta_log = store->meta_mgr->get_log(period_id);
6180 formatter->open_array_section("entries");
6181 for (; i < g_ceph_context->_conf->rgw_md_log_max_shards; i++) {
6183 list<cls_log_entry> entries;
6186 meta_log->init_list_entries(i, start_time.to_real_time(), end_time.to_real_time(), marker, &handle);
6189 int ret = meta_log->list_entries(handle, 1000, entries, NULL, &truncated);
6191 cerr << "ERROR: meta_log->list_entries(): " << cpp_strerror(-ret) << std::endl;
6195 for (list<cls_log_entry>::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
6196 cls_log_entry& entry = *iter;
6197 store->meta_mgr->dump_log_entry(entry, formatter);
6199 formatter->flush(cout);
6200 } while (truncated);
6202 meta_log->complete_list_entries(handle);
6204 if (specified_shard_id)
6209 formatter->close_section();
6210 formatter->flush(cout);
6213 if (opt_cmd == OPT_MDLOG_STATUS) {
6214 int i = (specified_shard_id ? shard_id : 0);
6216 if (period_id.empty()) {
6217 int ret = read_current_period_id(store, realm_id, realm_name, &period_id);
6221 std::cerr << "No --period given, using current period="
6222 << period_id << std::endl;
6224 RGWMetadataLog *meta_log = store->meta_mgr->get_log(period_id);
6226 formatter->open_array_section("entries");
6228 for (; i < g_ceph_context->_conf->rgw_md_log_max_shards; i++) {
6229 RGWMetadataLogInfo info;
6230 meta_log->get_info(i, &info);
6232 ::encode_json("info", info, formatter);
6234 if (specified_shard_id)
6239 formatter->close_section();
6240 formatter->flush(cout);
6243 if (opt_cmd == OPT_MDLOG_AUTOTRIM) {
6244 // need a full history for purging old mdlog periods
6245 store->meta_mgr->init_oldest_log_period();
6247 RGWCoroutinesManager crs(store->ctx(), store->get_cr_registry());
6248 RGWHTTPManager http(store->ctx(), crs.get_completion_mgr());
6249 int ret = http.set_threaded();
6251 cerr << "failed to initialize http client with " << cpp_strerror(ret) << std::endl;
6255 auto num_shards = g_conf->rgw_md_log_max_shards;
6256 ret = crs.run(create_admin_meta_log_trim_cr(store, &http, num_shards));
6258 cerr << "automated mdlog trim failed with " << cpp_strerror(ret) << std::endl;
6263 if (opt_cmd == OPT_MDLOG_TRIM) {
6264 utime_t start_time, end_time;
6266 if (!specified_shard_id) {
6267 cerr << "ERROR: shard-id must be specified for trim operation" << std::endl;
6271 int ret = parse_date_str(start_date, start_time);
6275 ret = parse_date_str(end_date, end_time);
6279 if (period_id.empty()) {
6280 std::cerr << "missing --period argument" << std::endl;
6283 RGWMetadataLog *meta_log = store->meta_mgr->get_log(period_id);
6285 ret = meta_log->trim(shard_id, start_time.to_real_time(), end_time.to_real_time(), start_marker, end_marker);
6287 cerr << "ERROR: meta_log->trim(): " << cpp_strerror(-ret) << std::endl;
6292 if (opt_cmd == OPT_SYNC_STATUS) {
6293 sync_status(formatter);
6296 if (opt_cmd == OPT_METADATA_SYNC_STATUS) {
6297 RGWMetaSyncStatusManager sync(store, store->get_async_rados());
6299 int ret = sync.init();
6301 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6305 rgw_meta_sync_status sync_status;
6306 ret = sync.read_sync_status(&sync_status);
6308 cerr << "ERROR: sync.read_sync_status() returned ret=" << ret << std::endl;
6312 formatter->open_object_section("summary");
6313 encode_json("sync_status", sync_status, formatter);
6315 uint64_t full_total = 0;
6316 uint64_t full_complete = 0;
6318 for (auto marker_iter : sync_status.sync_markers) {
6319 full_total += marker_iter.second.total_entries;
6320 if (marker_iter.second.state == rgw_meta_sync_marker::SyncState::FullSync) {
6321 full_complete += marker_iter.second.pos;
6323 full_complete += marker_iter.second.total_entries;
6327 formatter->open_object_section("full_sync");
6328 encode_json("total", full_total, formatter);
6329 encode_json("complete", full_complete, formatter);
6330 formatter->close_section();
6331 formatter->close_section();
6333 formatter->flush(cout);
6337 if (opt_cmd == OPT_METADATA_SYNC_INIT) {
6338 RGWMetaSyncStatusManager sync(store, store->get_async_rados());
6340 int ret = sync.init();
6342 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6345 ret = sync.init_sync_status();
6347 cerr << "ERROR: sync.init_sync_status() returned ret=" << ret << std::endl;
6353 if (opt_cmd == OPT_METADATA_SYNC_RUN) {
6354 RGWMetaSyncStatusManager sync(store, store->get_async_rados());
6356 int ret = sync.init();
6358 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6364 cerr << "ERROR: sync.run() returned ret=" << ret << std::endl;
6369 if (opt_cmd == OPT_DATA_SYNC_STATUS) {
6370 if (source_zone.empty()) {
6371 cerr << "ERROR: source zone not specified" << std::endl;
6374 RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone);
6376 int ret = sync.init();
6378 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6382 rgw_data_sync_status sync_status;
6383 ret = sync.read_sync_status(&sync_status);
6384 if (ret < 0 && ret != -ENOENT) {
6385 cerr << "ERROR: sync.read_sync_status() returned ret=" << ret << std::endl;
6389 formatter->open_object_section("summary");
6390 encode_json("sync_status", sync_status, formatter);
6392 uint64_t full_total = 0;
6393 uint64_t full_complete = 0;
6395 for (auto marker_iter : sync_status.sync_markers) {
6396 full_total += marker_iter.second.total_entries;
6397 if (marker_iter.second.state == rgw_meta_sync_marker::SyncState::FullSync) {
6398 full_complete += marker_iter.second.pos;
6400 full_complete += marker_iter.second.total_entries;
6404 formatter->open_object_section("full_sync");
6405 encode_json("total", full_total, formatter);
6406 encode_json("complete", full_complete, formatter);
6407 formatter->close_section();
6408 formatter->close_section();
6410 formatter->flush(cout);
6413 if (opt_cmd == OPT_DATA_SYNC_INIT) {
6414 if (source_zone.empty()) {
6415 cerr << "ERROR: source zone not specified" << std::endl;
6418 RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone);
6420 int ret = sync.init();
6422 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6426 ret = sync.init_sync_status();
6428 cerr << "ERROR: sync.init_sync_status() returned ret=" << ret << std::endl;
6433 if (opt_cmd == OPT_DATA_SYNC_RUN) {
6434 if (source_zone.empty()) {
6435 cerr << "ERROR: source zone not specified" << std::endl;
6438 RGWDataSyncStatusManager sync(store, store->get_async_rados(), source_zone);
6440 int ret = sync.init();
6442 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6448 cerr << "ERROR: sync.run() returned ret=" << ret << std::endl;
6453 if (opt_cmd == OPT_BUCKET_SYNC_INIT) {
6454 if (source_zone.empty()) {
6455 cerr << "ERROR: source zone not specified" << std::endl;
6458 if (bucket_name.empty()) {
6459 cerr << "ERROR: bucket not specified" << std::endl;
6463 int ret = init_bucket_for_sync(tenant, bucket_name, bucket_id, bucket);
6467 RGWBucketSyncStatusManager sync(store, source_zone, bucket);
6471 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6474 ret = sync.init_sync_status();
6476 cerr << "ERROR: sync.init_sync_status() returned ret=" << ret << std::endl;
6481 if ((opt_cmd == OPT_BUCKET_SYNC_DISABLE) || (opt_cmd == OPT_BUCKET_SYNC_ENABLE)) {
6482 if (bucket_name.empty()) {
6483 cerr << "ERROR: bucket not specified" << std::endl;
6488 cerr << "could not init realm " << ": " << cpp_strerror(-ret) << std::endl;
6492 ret = period.init(g_ceph_context, store, realm_id, realm_name, true);
6494 cerr << "failed to init period " << ": " << cpp_strerror(-ret) << std::endl;
6498 if (!store->is_meta_master()) {
6499 cerr << "failed to update bucket sync: only allowed on meta master zone " << std::endl;
6500 cerr << period.get_master_zone() << " | " << period.get_realm() << std::endl;
6504 rgw_obj obj(bucket, object);
6505 ret = set_bucket_sync_enabled(store, opt_cmd, tenant, bucket_name);
6510 if (opt_cmd == OPT_BUCKET_SYNC_STATUS) {
6511 if (source_zone.empty()) {
6512 cerr << "ERROR: source zone not specified" << std::endl;
6515 if (bucket_name.empty()) {
6516 cerr << "ERROR: bucket not specified" << std::endl;
6520 int ret = init_bucket_for_sync(tenant, bucket_name, bucket_id, bucket);
6524 RGWBucketSyncStatusManager sync(store, source_zone, bucket);
6528 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6531 ret = sync.read_sync_status();
6533 cerr << "ERROR: sync.read_sync_status() returned ret=" << ret << std::endl;
6537 map<int, rgw_bucket_shard_sync_info>& sync_status = sync.get_sync_status();
6539 encode_json("sync_status", sync_status, formatter);
6540 formatter->flush(cout);
6543 if (opt_cmd == OPT_BUCKET_SYNC_RUN) {
6544 if (source_zone.empty()) {
6545 cerr << "ERROR: source zone not specified" << std::endl;
6548 if (bucket_name.empty()) {
6549 cerr << "ERROR: bucket not specified" << std::endl;
6553 int ret = init_bucket_for_sync(tenant, bucket_name, bucket_id, bucket);
6557 RGWBucketSyncStatusManager sync(store, source_zone, bucket);
6561 cerr << "ERROR: sync.init() returned ret=" << ret << std::endl;
6567 cerr << "ERROR: sync.run() returned ret=" << ret << std::endl;
6572 if (opt_cmd == OPT_BILOG_LIST) {
6573 if (bucket_name.empty()) {
6574 cerr << "ERROR: bucket not specified" << std::endl;
6577 RGWBucketInfo bucket_info;
6578 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
6580 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
6583 formatter->open_array_section("entries");
6586 if (max_entries < 0)
6590 list<rgw_bi_log_entry> entries;
6591 ret = store->list_bi_log_entries(bucket_info, shard_id, marker, max_entries - count, entries, &truncated);
6593 cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
6597 count += entries.size();
6599 for (list<rgw_bi_log_entry>::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
6600 rgw_bi_log_entry& entry = *iter;
6601 encode_json("entry", entry, formatter);
6605 formatter->flush(cout);
6606 } while (truncated && count < max_entries);
6608 formatter->close_section();
6609 formatter->flush(cout);
6612 if (opt_cmd == OPT_SYNC_ERROR_LIST) {
6613 if (max_entries < 0) {
6618 utime_t start_time, end_time;
6620 int ret = parse_date_str(start_date, start_time);
6624 ret = parse_date_str(end_date, end_time);
6632 formatter->open_array_section("entries");
6634 for (; shard_id < ERROR_LOGGER_SHARDS; ++shard_id) {
6635 formatter->open_object_section("shard");
6636 encode_json("shard_id", shard_id, formatter);
6637 formatter->open_array_section("entries");
6640 string oid = RGWSyncErrorLogger::get_shard_oid(RGW_SYNC_ERROR_LOG_SHARD_PREFIX, shard_id);
6643 list<cls_log_entry> entries;
6644 ret = store->time_log_list(oid, start_time.to_real_time(), end_time.to_real_time(),
6645 max_entries - count, entries, marker, &marker, &truncated);
6646 if (ret == -ENOENT) {
6650 cerr << "ERROR: store->time_log_list(): " << cpp_strerror(-ret) << std::endl;
6654 count += entries.size();
6656 for (auto& cls_entry : entries) {
6657 rgw_sync_error_info log_entry;
6659 auto iter = cls_entry.data.begin();
6661 ::decode(log_entry, iter);
6662 } catch (buffer::error& err) {
6663 cerr << "ERROR: failed to decode log entry" << std::endl;
6666 formatter->open_object_section("entry");
6667 encode_json("id", cls_entry.id, formatter);
6668 encode_json("section", cls_entry.section, formatter);
6669 encode_json("name", cls_entry.name, formatter);
6670 encode_json("timestamp", cls_entry.timestamp, formatter);
6671 encode_json("info", log_entry, formatter);
6672 formatter->close_section();
6673 formatter->flush(cout);
6675 } while (truncated && count < max_entries);
6677 formatter->close_section();
6678 formatter->close_section();
6680 if (specified_shard_id) {
6685 formatter->close_section();
6686 formatter->flush(cout);
6689 if (opt_cmd == OPT_BILOG_TRIM) {
6690 if (bucket_name.empty()) {
6691 cerr << "ERROR: bucket not specified" << std::endl;
6694 RGWBucketInfo bucket_info;
6695 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
6697 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
6700 ret = store->trim_bi_log_entries(bucket_info, shard_id, start_marker, end_marker);
6702 cerr << "ERROR: trim_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
6707 if (opt_cmd == OPT_BILOG_STATUS) {
6708 if (bucket_name.empty()) {
6709 cerr << "ERROR: bucket not specified" << std::endl;
6712 RGWBucketInfo bucket_info;
6713 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
6715 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
6718 map<int, string> markers;
6719 ret = store->get_bi_log_status(bucket_info, shard_id, markers);
6721 cerr << "ERROR: get_bi_log_status(): " << cpp_strerror(-ret) << std::endl;
6724 formatter->open_object_section("entries");
6725 encode_json("markers", markers, formatter);
6726 formatter->close_section();
6727 formatter->flush(cout);
6730 if (opt_cmd == OPT_DATALOG_LIST) {
6731 formatter->open_array_section("entries");
6734 if (max_entries < 0)
6737 utime_t start_time, end_time;
6739 int ret = parse_date_str(start_date, start_time);
6743 ret = parse_date_str(end_date, end_time);
6747 RGWDataChangesLog *log = store->data_log;
6748 RGWDataChangesLog::LogMarker marker;
6751 list<rgw_data_change_log_entry> entries;
6752 ret = log->list_entries(start_time.to_real_time(), end_time.to_real_time(), max_entries - count, entries, marker, &truncated);
6754 cerr << "ERROR: list_bi_log_entries(): " << cpp_strerror(-ret) << std::endl;
6758 count += entries.size();
6760 for (list<rgw_data_change_log_entry>::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
6761 rgw_data_change_log_entry& entry = *iter;
6763 encode_json("entry", entry.entry, formatter);
6765 encode_json("entry", entry, formatter);
6768 formatter->flush(cout);
6769 } while (truncated && count < max_entries);
6771 formatter->close_section();
6772 formatter->flush(cout);
6775 if (opt_cmd == OPT_DATALOG_STATUS) {
6776 RGWDataChangesLog *log = store->data_log;
6777 int i = (specified_shard_id ? shard_id : 0);
6779 formatter->open_array_section("entries");
6780 for (; i < g_ceph_context->_conf->rgw_data_log_num_shards; i++) {
6781 list<cls_log_entry> entries;
6783 RGWDataChangesLogInfo info;
6784 log->get_info(i, &info);
6786 ::encode_json("info", info, formatter);
6788 if (specified_shard_id)
6792 formatter->close_section();
6793 formatter->flush(cout);
6796 if (opt_cmd == OPT_DATALOG_TRIM) {
6797 utime_t start_time, end_time;
6799 int ret = parse_date_str(start_date, start_time);
6803 ret = parse_date_str(end_date, end_time);
6807 RGWDataChangesLog *log = store->data_log;
6808 ret = log->trim_entries(start_time.to_real_time(), end_time.to_real_time(), start_marker, end_marker);
6810 cerr << "ERROR: trim_entries(): " << cpp_strerror(-ret) << std::endl;
6815 if (opt_cmd == OPT_OPSTATE_LIST) {
6816 RGWOpState oc(store);
6821 oc.init_list_entries(client_id, op_id, object, &handle);
6822 list<cls_statelog_entry> entries;
6824 formatter->open_array_section("entries");
6826 int ret = oc.list_entries(handle, max, entries, &done);
6828 cerr << "oc.list_entries returned " << cpp_strerror(-ret) << std::endl;
6829 oc.finish_list_entries(handle);
6833 for (list<cls_statelog_entry>::iterator iter = entries.begin(); iter != entries.end(); ++iter) {
6834 oc.dump_entry(*iter, formatter);
6837 formatter->flush(cout);
6839 formatter->close_section();
6840 formatter->flush(cout);
6841 oc.finish_list_entries(handle);
6844 if (opt_cmd == OPT_OPSTATE_SET || opt_cmd == OPT_OPSTATE_RENEW) {
6845 RGWOpState oc(store);
6847 RGWOpState::OpState state;
6848 if (object.empty() || client_id.empty() || op_id.empty()) {
6849 cerr << "ERROR: need to specify client_id, op_id, and object" << std::endl;
6852 if (state_str.empty()) {
6853 cerr << "ERROR: state was not specified" << std::endl;
6856 int ret = oc.state_from_str(state_str, &state);
6858 cerr << "ERROR: invalid state: " << state_str << std::endl;
6862 if (opt_cmd == OPT_OPSTATE_SET) {
6863 ret = oc.set_state(client_id, op_id, object, state);
6865 cerr << "ERROR: failed to set state: " << cpp_strerror(-ret) << std::endl;
6869 ret = oc.renew_state(client_id, op_id, object, state);
6871 cerr << "ERROR: failed to renew state: " << cpp_strerror(-ret) << std::endl;
6876 if (opt_cmd == OPT_OPSTATE_RM) {
6877 RGWOpState oc(store);
6879 if (object.empty() || client_id.empty() || op_id.empty()) {
6880 cerr << "ERROR: need to specify client_id, op_id, and object" << std::endl;
6883 ret = oc.remove_entry(client_id, op_id, object);
6885 cerr << "ERROR: failed to set state: " << cpp_strerror(-ret) << std::endl;
6890 if (opt_cmd == OPT_REPLICALOG_GET || opt_cmd == OPT_REPLICALOG_UPDATE ||
6891 opt_cmd == OPT_REPLICALOG_DELETE) {
6892 if (replica_log_type_str.empty()) {
6893 cerr << "ERROR: need to specify --replica-log-type=<metadata | data | bucket>" << std::endl;
6898 if (opt_cmd == OPT_REPLICALOG_GET) {
6899 RGWReplicaBounds bounds;
6900 if (replica_log_type == ReplicaLog_Metadata) {
6901 if (!specified_shard_id) {
6902 cerr << "ERROR: shard-id must be specified for get operation" << std::endl;
6906 RGWReplicaObjectLogger logger(store, pool, META_REPLICA_LOG_OBJ_PREFIX);
6907 int ret = logger.get_bounds(shard_id, bounds);
6910 } else if (replica_log_type == ReplicaLog_Data) {
6911 if (!specified_shard_id) {
6912 cerr << "ERROR: shard-id must be specified for get operation" << std::endl;
6915 RGWReplicaObjectLogger logger(store, pool, DATA_REPLICA_LOG_OBJ_PREFIX);
6916 int ret = logger.get_bounds(shard_id, bounds);
6919 } else if (replica_log_type == ReplicaLog_Bucket) {
6920 if (bucket_name.empty()) {
6921 cerr << "ERROR: bucket not specified" << std::endl;
6924 RGWBucketInfo bucket_info;
6925 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
6927 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
6931 RGWReplicaBucketLogger logger(store);
6932 ret = logger.get_bounds(bucket, shard_id, bounds);
6935 } else { // shouldn't get here
6938 encode_json("bounds", bounds, formatter);
6939 formatter->flush(cout);
6943 if (opt_cmd == OPT_REPLICALOG_DELETE) {
6944 if (replica_log_type == ReplicaLog_Metadata) {
6945 if (!specified_shard_id) {
6946 cerr << "ERROR: shard-id must be specified for delete operation" << std::endl;
6949 if (!specified_daemon_id) {
6950 cerr << "ERROR: daemon-id must be specified for delete operation" << std::endl;
6953 RGWReplicaObjectLogger logger(store, pool, META_REPLICA_LOG_OBJ_PREFIX);
6954 int ret = logger.delete_bound(shard_id, daemon_id, false);
6957 } else if (replica_log_type == ReplicaLog_Data) {
6958 if (!specified_shard_id) {
6959 cerr << "ERROR: shard-id must be specified for delete operation" << std::endl;
6962 if (!specified_daemon_id) {
6963 cerr << "ERROR: daemon-id must be specified for delete operation" << std::endl;
6966 RGWReplicaObjectLogger logger(store, pool, DATA_REPLICA_LOG_OBJ_PREFIX);
6967 int ret = logger.delete_bound(shard_id, daemon_id, false);
6970 } else if (replica_log_type == ReplicaLog_Bucket) {
6971 if (bucket_name.empty()) {
6972 cerr << "ERROR: bucket not specified" << std::endl;
6975 RGWBucketInfo bucket_info;
6976 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
6978 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
6982 RGWReplicaBucketLogger logger(store);
6983 ret = logger.delete_bound(bucket, shard_id, daemon_id, false);
6989 if (opt_cmd == OPT_REPLICALOG_UPDATE) {
6990 if (marker.empty()) {
6991 cerr << "ERROR: marker was not specified" <<std::endl;
6994 utime_t time = ceph_clock_now();
6995 if (!date.empty()) {
6996 ret = parse_date_str(date, time);
6998 cerr << "ERROR: failed to parse start date" << std::endl;
7002 list<RGWReplicaItemMarker> entries;
7003 int ret = read_decode_json(infile, entries);
7005 cerr << "ERROR: failed to decode entries" << std::endl;
7008 RGWReplicaBounds bounds;
7009 if (replica_log_type == ReplicaLog_Metadata) {
7010 if (!specified_shard_id) {
7011 cerr << "ERROR: shard-id must be specified for get operation" << std::endl;
7015 RGWReplicaObjectLogger logger(store, pool, META_REPLICA_LOG_OBJ_PREFIX);
7016 int ret = logger.update_bound(shard_id, daemon_id, marker, time, &entries);
7018 cerr << "ERROR: failed to update bounds: " << cpp_strerror(-ret) << std::endl;
7021 } else if (replica_log_type == ReplicaLog_Data) {
7022 if (!specified_shard_id) {
7023 cerr << "ERROR: shard-id must be specified for get operation" << std::endl;
7026 RGWReplicaObjectLogger logger(store, pool, DATA_REPLICA_LOG_OBJ_PREFIX);
7027 int ret = logger.update_bound(shard_id, daemon_id, marker, time, &entries);
7029 cerr << "ERROR: failed to update bounds: " << cpp_strerror(-ret) << std::endl;
7032 } else if (replica_log_type == ReplicaLog_Bucket) {
7033 if (bucket_name.empty()) {
7034 cerr << "ERROR: bucket not specified" << std::endl;
7037 RGWBucketInfo bucket_info;
7038 int ret = init_bucket(tenant, bucket_name, bucket_id, bucket_info, bucket);
7040 cerr << "ERROR: could not init bucket: " << cpp_strerror(-ret) << std::endl;
7044 RGWReplicaBucketLogger logger(store);
7045 ret = logger.update_bound(bucket, shard_id, daemon_id, marker, time, &entries);
7047 cerr << "ERROR: failed to update bounds: " << cpp_strerror(-ret) << std::endl;
7053 bool quota_op = (opt_cmd == OPT_QUOTA_SET || opt_cmd == OPT_QUOTA_ENABLE || opt_cmd == OPT_QUOTA_DISABLE);
7056 if (bucket_name.empty() && user_id.empty()) {
7057 cerr << "ERROR: bucket name or uid is required for quota operation" << std::endl;
7061 if (!bucket_name.empty()) {
7062 if (!quota_scope.empty() && quota_scope != "bucket") {
7063 cerr << "ERROR: invalid quota scope specification." << std::endl;
7066 set_bucket_quota(store, opt_cmd, tenant, bucket_name,
7067 max_size, max_objects, have_max_size, have_max_objects);
7068 } else if (!user_id.empty()) {
7069 if (quota_scope == "bucket") {
7070 set_user_bucket_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
7071 } else if (quota_scope == "user") {
7072 set_user_quota(opt_cmd, user, user_op, max_size, max_objects, have_max_size, have_max_objects);
7074 cerr << "ERROR: invalid quota scope specification. Please specify either --quota-scope=bucket, or --quota-scope=user" << std::endl;