kernel/drivers/staging/lustre/lnet/selftest/console.c

   1 /*
   2  * GPL HEADER START
   3  *
   4  * DO NOT ALTER OR REMOVE COPYRIGHT NOTICES OR THIS FILE HEADER.
   5  *
   6  * This program is free software; you can redistribute it and/or modify
   7  * it under the terms of the GNU General Public License version 2 only,
   8  * as published by the Free Software Foundation.
   9  *
  10  * This program is distributed in the hope that it will be useful, but
  11  * WITHOUT ANY WARRANTY; without even the implied warranty of
  12  * MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE.  See the GNU
  13  * General Public License version 2 for more details (a copy is included
  14  * in the LICENSE file that accompanied this code).
  15  *
  16  * You should have received a copy of the GNU General Public License
  17  * version 2 along with this program; If not, see
  18  * http://www.sun.com/software/products/lustre/docs/GPLv2.pdf
  19  *
  20  * Please contact Sun Microsystems, Inc., 4150 Network Circle, Santa Clara,
  21  * CA 95054 USA or visit www.sun.com if you need additional information or
  22  * have any questions.
  23  *
  24  * GPL HEADER END
  25  */
  26 /*
  27  * Copyright (c) 2007, 2010, Oracle and/or its affiliates. All rights reserved.
  28  * Use is subject to license terms.
  29  *
  30  * Copyright (c) 2012, Intel Corporation.
  31  */
  32 /*
  33  * This file is part of Lustre, http://www.lustre.org/
  34  * Lustre is a trademark of Sun Microsystems, Inc.
  35  *
  36  * lnet/selftest/conctl.c
  37  *
  38  * Infrastructure of LST console
  39  *
  40  * Author: Liang Zhen <liangzhen@clusterfs.com>
  41  */
  42
  43 #include "../../include/linux/libcfs/libcfs.h"
  44 #include "../../include/linux/lnet/lib-lnet.h"
  45 #include "console.h"
  46 #include "conrpc.h"
  47
  48 #define LST_NODE_STATE_COUNTER(nd, p)                   \
  49 do {                                                    \
  50         if ((nd)->nd_state == LST_NODE_ACTIVE)          \
  51                 (p)->nle_nactive++;                     \
  52         else if ((nd)->nd_state == LST_NODE_BUSY)       \
  53                 (p)->nle_nbusy++;                       \
  54         else if ((nd)->nd_state == LST_NODE_DOWN)       \
  55                 (p)->nle_ndown++;                       \
  56         else                                            \
  57                 (p)->nle_nunknown++;                    \
  58         (p)->nle_nnode++;                               \
  59 } while (0)
  60
  61 lstcon_session_t console_session;
  62
  63 static void
  64 lstcon_node_get(lstcon_node_t *nd)
  65 {
  66         LASSERT(nd->nd_ref >= 1);
  67
  68         nd->nd_ref++;
  69 }
  70
  71 static int
  72 lstcon_node_find(lnet_process_id_t id, lstcon_node_t **ndpp, int create)
  73 {
  74         lstcon_ndlink_t *ndl;
  75         unsigned int idx = LNET_NIDADDR(id.nid) % LST_GLOBAL_HASHSIZE;
  76
  77         LASSERT(id.nid != LNET_NID_ANY);
  78
  79         list_for_each_entry(ndl, &console_session.ses_ndl_hash[idx], ndl_hlink) {
  80                 if (ndl->ndl_node->nd_id.nid != id.nid ||
  81                     ndl->ndl_node->nd_id.pid != id.pid)
  82                         continue;
  83
  84                 lstcon_node_get(ndl->ndl_node);
  85                 *ndpp = ndl->ndl_node;
  86                 return 0;
  87         }
  88
  89         if (!create)
  90                 return -ENOENT;
  91
  92         LIBCFS_ALLOC(*ndpp, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
  93         if (*ndpp == NULL)
  94                 return -ENOMEM;
  95
  96         ndl = (lstcon_ndlink_t *)(*ndpp + 1);
  97
  98         ndl->ndl_node = *ndpp;
  99
 100         ndl->ndl_node->nd_ref   = 1;
 101         ndl->ndl_node->nd_id    = id;
 102         ndl->ndl_node->nd_stamp = cfs_time_current();
 103         ndl->ndl_node->nd_state = LST_NODE_UNKNOWN;
 104         ndl->ndl_node->nd_timeout = 0;
 105         memset(&ndl->ndl_node->nd_ping, 0, sizeof(lstcon_rpc_t));
 106
 107         /* queued in global hash & list, no refcount is taken by
 108          * global hash & list, if caller release his refcount,
 109          * node will be released */
 110         list_add_tail(&ndl->ndl_hlink, &console_session.ses_ndl_hash[idx]);
 111         list_add_tail(&ndl->ndl_link, &console_session.ses_ndl_list);
 112
 113         return 0;
 114 }
 115
 116 static void
 117 lstcon_node_put(lstcon_node_t *nd)
 118 {
 119         lstcon_ndlink_t *ndl;
 120
 121         LASSERT(nd->nd_ref > 0);
 122
 123         if (--nd->nd_ref > 0)
 124                 return;
 125
 126         ndl = (lstcon_ndlink_t *)(nd + 1);
 127
 128         LASSERT(!list_empty(&ndl->ndl_link));
 129         LASSERT(!list_empty(&ndl->ndl_hlink));
 130
 131         /* remove from session */
 132         list_del(&ndl->ndl_link);
 133         list_del(&ndl->ndl_hlink);
 134
 135         LIBCFS_FREE(nd, sizeof(lstcon_node_t) + sizeof(lstcon_ndlink_t));
 136 }
 137
 138 static int
 139 lstcon_ndlink_find(struct list_head *hash,
 140                    lnet_process_id_t id, lstcon_ndlink_t **ndlpp, int create)
 141 {
 142         unsigned int idx = LNET_NIDADDR(id.nid) % LST_NODE_HASHSIZE;
 143         lstcon_ndlink_t *ndl;
 144         lstcon_node_t *nd;
 145         int rc;
 146
 147         if (id.nid == LNET_NID_ANY)
 148                 return -EINVAL;
 149
 150         /* search in hash */
 151         list_for_each_entry(ndl, &hash[idx], ndl_hlink) {
 152                 if (ndl->ndl_node->nd_id.nid != id.nid ||
 153                     ndl->ndl_node->nd_id.pid != id.pid)
 154                         continue;
 155
 156                 *ndlpp = ndl;
 157                 return 0;
 158         }
 159
 160         if (create == 0)
 161                 return -ENOENT;
 162
 163         /* find or create in session hash */
 164         rc = lstcon_node_find(id, &nd, (create == 1) ? 1 : 0);
 165         if (rc != 0)
 166                 return rc;
 167
 168         LIBCFS_ALLOC(ndl, sizeof(lstcon_ndlink_t));
 169         if (ndl == NULL) {
 170                 lstcon_node_put(nd);
 171                 return -ENOMEM;
 172         }
 173
 174         *ndlpp = ndl;
 175
 176         ndl->ndl_node = nd;
 177         INIT_LIST_HEAD(&ndl->ndl_link);
 178         list_add_tail(&ndl->ndl_hlink, &hash[idx]);
 179
 180         return  0;
 181 }
 182
 183 static void
 184 lstcon_ndlink_release(lstcon_ndlink_t *ndl)
 185 {
 186         LASSERT(list_empty(&ndl->ndl_link));
 187         LASSERT(!list_empty(&ndl->ndl_hlink));
 188
 189         list_del(&ndl->ndl_hlink); /* delete from hash */
 190         lstcon_node_put(ndl->ndl_node);
 191
 192         LIBCFS_FREE(ndl, sizeof(*ndl));
 193 }
 194
 195 static int
 196 lstcon_group_alloc(char *name, lstcon_group_t **grpp)
 197 {
 198         lstcon_group_t *grp;
 199         int i;
 200
 201         LIBCFS_ALLOC(grp, offsetof(lstcon_group_t,
 202                                    grp_ndl_hash[LST_NODE_HASHSIZE]));
 203         if (grp == NULL)
 204                 return -ENOMEM;
 205
 206         grp->grp_ref = 1;
 207         if (name != NULL)
 208                 strcpy(grp->grp_name, name);
 209
 210         INIT_LIST_HEAD(&grp->grp_link);
 211         INIT_LIST_HEAD(&grp->grp_ndl_list);
 212         INIT_LIST_HEAD(&grp->grp_trans_list);
 213
 214         for (i = 0; i < LST_NODE_HASHSIZE; i++)
 215                 INIT_LIST_HEAD(&grp->grp_ndl_hash[i]);
 216
 217         *grpp = grp;
 218
 219         return 0;
 220 }
 221
 222 static void
 223 lstcon_group_addref(lstcon_group_t *grp)
 224 {
 225         grp->grp_ref++;
 226 }
 227
 228 static void lstcon_group_ndlink_release(lstcon_group_t *, lstcon_ndlink_t *);
 229
 230 static void
 231 lstcon_group_drain(lstcon_group_t *grp, int keep)
 232 {
 233         lstcon_ndlink_t *ndl;
 234         lstcon_ndlink_t *tmp;
 235
 236         list_for_each_entry_safe(ndl, tmp, &grp->grp_ndl_list, ndl_link) {
 237                 if ((ndl->ndl_node->nd_state & keep) == 0)
 238                         lstcon_group_ndlink_release(grp, ndl);
 239         }
 240 }
 241
 242 static void
 243 lstcon_group_decref(lstcon_group_t *grp)
 244 {
 245         int i;
 246
 247         if (--grp->grp_ref > 0)
 248                 return;
 249
 250         if (!list_empty(&grp->grp_link))
 251                 list_del(&grp->grp_link);
 252
 253         lstcon_group_drain(grp, 0);
 254
 255         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 256                 LASSERT(list_empty(&grp->grp_ndl_hash[i]));
 257         }
 258
 259         LIBCFS_FREE(grp, offsetof(lstcon_group_t,
 260                                   grp_ndl_hash[LST_NODE_HASHSIZE]));
 261 }
 262
 263 static int
 264 lstcon_group_find(const char *name, lstcon_group_t **grpp)
 265 {
 266         lstcon_group_t *grp;
 267
 268         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 269                 if (strncmp(grp->grp_name, name, LST_NAME_SIZE) != 0)
 270                         continue;
 271
 272                 lstcon_group_addref(grp);  /* +1 ref for caller */
 273                 *grpp = grp;
 274                 return 0;
 275         }
 276
 277         return -ENOENT;
 278 }
 279
 280 static void
 281 lstcon_group_put(lstcon_group_t *grp)
 282 {
 283         lstcon_group_decref(grp);
 284 }
 285
 286 static int
 287 lstcon_group_ndlink_find(lstcon_group_t *grp, lnet_process_id_t id,
 288                          lstcon_ndlink_t **ndlpp, int create)
 289 {
 290         int rc;
 291
 292         rc = lstcon_ndlink_find(&grp->grp_ndl_hash[0], id, ndlpp, create);
 293         if (rc != 0)
 294                 return rc;
 295
 296         if (!list_empty(&(*ndlpp)->ndl_link))
 297                 return 0;
 298
 299         list_add_tail(&(*ndlpp)->ndl_link, &grp->grp_ndl_list);
 300         grp->grp_nnode++;
 301
 302         return 0;
 303 }
 304
 305 static void
 306 lstcon_group_ndlink_release(lstcon_group_t *grp, lstcon_ndlink_t *ndl)
 307 {
 308         list_del_init(&ndl->ndl_link);
 309         lstcon_ndlink_release(ndl);
 310         grp->grp_nnode--;
 311 }
 312
 313 static void
 314 lstcon_group_ndlink_move(lstcon_group_t *old,
 315                          lstcon_group_t *new, lstcon_ndlink_t *ndl)
 316 {
 317         unsigned int idx = LNET_NIDADDR(ndl->ndl_node->nd_id.nid) %
 318                            LST_NODE_HASHSIZE;
 319
 320         list_del(&ndl->ndl_hlink);
 321         list_del(&ndl->ndl_link);
 322         old->grp_nnode--;
 323
 324         list_add_tail(&ndl->ndl_hlink, &new->grp_ndl_hash[idx]);
 325         list_add_tail(&ndl->ndl_link, &new->grp_ndl_list);
 326         new->grp_nnode++;
 327
 328         return;
 329 }
 330
 331 static void
 332 lstcon_group_move(lstcon_group_t *old, lstcon_group_t *new)
 333 {
 334         lstcon_ndlink_t *ndl;
 335
 336         while (!list_empty(&old->grp_ndl_list)) {
 337                 ndl = list_entry(old->grp_ndl_list.next,
 338                                      lstcon_ndlink_t, ndl_link);
 339                 lstcon_group_ndlink_move(old, new, ndl);
 340         }
 341 }
 342
 343 static int
 344 lstcon_sesrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 345 {
 346         lstcon_group_t *grp = (lstcon_group_t *)arg;
 347
 348         switch (transop) {
 349         case LST_TRANS_SESNEW:
 350                 if (nd->nd_state == LST_NODE_ACTIVE)
 351                         return 0;
 352                 break;
 353
 354         case LST_TRANS_SESEND:
 355                 if (nd->nd_state != LST_NODE_ACTIVE)
 356                         return 0;
 357
 358                 if (grp != NULL && nd->nd_ref > 1)
 359                         return 0;
 360                 break;
 361
 362         case LST_TRANS_SESQRY:
 363                 break;
 364
 365         default:
 366                 LBUG();
 367         }
 368
 369         return 1;
 370 }
 371
 372 static int
 373 lstcon_sesrpc_readent(int transop, srpc_msg_t *msg,
 374                       lstcon_rpc_ent_t *ent_up)
 375 {
 376         srpc_debug_reply_t *rep;
 377
 378         switch (transop) {
 379         case LST_TRANS_SESNEW:
 380         case LST_TRANS_SESEND:
 381                 return 0;
 382
 383         case LST_TRANS_SESQRY:
 384                 rep = &msg->msg_body.dbg_reply;
 385
 386                 if (copy_to_user(&ent_up->rpe_priv[0],
 387                                      &rep->dbg_timeout, sizeof(int)) ||
 388                     copy_to_user(&ent_up->rpe_payload[0],
 389                                      &rep->dbg_name, LST_NAME_SIZE))
 390                         return -EFAULT;
 391
 392                 return 0;
 393
 394         default:
 395                 LBUG();
 396         }
 397
 398         return 0;
 399 }
 400
 401 static int
 402 lstcon_group_nodes_add(lstcon_group_t *grp,
 403                        int count, lnet_process_id_t *ids_up,
 404                        unsigned *featp, struct list_head *result_up)
 405 {
 406         lstcon_rpc_trans_t *trans;
 407         lstcon_ndlink_t *ndl;
 408         lstcon_group_t *tmp;
 409         lnet_process_id_t id;
 410         int i;
 411         int rc;
 412
 413         rc = lstcon_group_alloc(NULL, &tmp);
 414         if (rc != 0) {
 415                 CERROR("Out of memory\n");
 416                 return -ENOMEM;
 417         }
 418
 419         for (i = 0 ; i < count; i++) {
 420                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 421                         rc = -EFAULT;
 422                         break;
 423                 }
 424
 425                 /* skip if it's in this group already */
 426                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 0);
 427                 if (rc == 0)
 428                         continue;
 429
 430                 /* add to tmp group */
 431                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 1);
 432                 if (rc != 0) {
 433                         CERROR("Can't create ndlink, out of memory\n");
 434                         break;
 435                 }
 436         }
 437
 438         if (rc != 0) {
 439                 lstcon_group_put(tmp);
 440                 return rc;
 441         }
 442
 443         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 444                                      &tmp->grp_trans_list, LST_TRANS_SESNEW,
 445                                      tmp, lstcon_sesrpc_condition, &trans);
 446         if (rc != 0) {
 447                 CERROR("Can't create transaction: %d\n", rc);
 448                 lstcon_group_put(tmp);
 449                 return rc;
 450         }
 451
 452         /* post all RPCs */
 453         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 454
 455         rc = lstcon_rpc_trans_interpreter(trans, result_up,
 456                                           lstcon_sesrpc_readent);
 457         *featp = trans->tas_features;
 458
 459         /* destroy all RPGs */
 460         lstcon_rpc_trans_destroy(trans);
 461
 462         lstcon_group_move(tmp, grp);
 463         lstcon_group_put(tmp);
 464
 465         return rc;
 466 }
 467
 468 static int
 469 lstcon_group_nodes_remove(lstcon_group_t *grp,
 470                           int count, lnet_process_id_t *ids_up,
 471                           struct list_head *result_up)
 472 {
 473         lstcon_rpc_trans_t *trans;
 474         lstcon_ndlink_t *ndl;
 475         lstcon_group_t *tmp;
 476         lnet_process_id_t id;
 477         int rc;
 478         int i;
 479
 480         /* End session and remove node from the group */
 481
 482         rc = lstcon_group_alloc(NULL, &tmp);
 483         if (rc != 0) {
 484                 CERROR("Out of memory\n");
 485                 return -ENOMEM;
 486         }
 487
 488         for (i = 0; i < count; i++) {
 489                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
 490                         rc = -EFAULT;
 491                         goto error;
 492                 }
 493
 494                 /* move node to tmp group */
 495                 if (lstcon_group_ndlink_find(grp, id, &ndl, 0) == 0)
 496                         lstcon_group_ndlink_move(grp, tmp, ndl);
 497         }
 498
 499         rc = lstcon_rpc_trans_ndlist(&tmp->grp_ndl_list,
 500                                      &tmp->grp_trans_list, LST_TRANS_SESEND,
 501                                      tmp, lstcon_sesrpc_condition, &trans);
 502         if (rc != 0) {
 503                 CERROR("Can't create transaction: %d\n", rc);
 504                 goto error;
 505         }
 506
 507         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 508
 509         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 510
 511         lstcon_rpc_trans_destroy(trans);
 512         /* release nodes anyway, because we can't rollback status */
 513         lstcon_group_put(tmp);
 514
 515         return rc;
 516 error:
 517         lstcon_group_move(tmp, grp);
 518         lstcon_group_put(tmp);
 519
 520         return rc;
 521 }
 522
 523 int
 524 lstcon_group_add(char *name)
 525 {
 526         lstcon_group_t *grp;
 527         int rc;
 528
 529         rc = (lstcon_group_find(name, &grp) == 0) ? -EEXIST : 0;
 530         if (rc != 0) {
 531                 /* find a group with same name */
 532                 lstcon_group_put(grp);
 533                 return rc;
 534         }
 535
 536         rc = lstcon_group_alloc(name, &grp);
 537         if (rc != 0) {
 538                 CERROR("Can't allocate descriptor for group %s\n", name);
 539                 return -ENOMEM;
 540         }
 541
 542         list_add_tail(&grp->grp_link, &console_session.ses_grp_list);
 543
 544         return rc;
 545 }
 546
 547 int
 548 lstcon_nodes_add(char *name, int count, lnet_process_id_t *ids_up,
 549                  unsigned *featp, struct list_head *result_up)
 550 {
 551         lstcon_group_t *grp;
 552         int rc;
 553
 554         LASSERT(count > 0);
 555         LASSERT(ids_up != NULL);
 556
 557         rc = lstcon_group_find(name, &grp);
 558         if (rc != 0) {
 559                 CDEBUG(D_NET, "Can't find group %s\n", name);
 560                 return rc;
 561         }
 562
 563         if (grp->grp_ref > 2) {
 564                 /* referred by other threads or test */
 565                 CDEBUG(D_NET, "Group %s is busy\n", name);
 566                 lstcon_group_put(grp);
 567
 568                 return -EBUSY;
 569         }
 570
 571         rc = lstcon_group_nodes_add(grp, count, ids_up, featp, result_up);
 572
 573         lstcon_group_put(grp);
 574
 575         return rc;
 576 }
 577
 578 int
 579 lstcon_group_del(char *name)
 580 {
 581         lstcon_rpc_trans_t *trans;
 582         lstcon_group_t *grp;
 583         int rc;
 584
 585         rc = lstcon_group_find(name, &grp);
 586         if (rc != 0) {
 587                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 588                 return rc;
 589         }
 590
 591         if (grp->grp_ref > 2) {
 592                 /* referred by others threads or test */
 593                 CDEBUG(D_NET, "Group %s is busy\n", name);
 594                 lstcon_group_put(grp);
 595                 return -EBUSY;
 596         }
 597
 598         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 599                                      &grp->grp_trans_list, LST_TRANS_SESEND,
 600                                      grp, lstcon_sesrpc_condition, &trans);
 601         if (rc != 0) {
 602                 CERROR("Can't create transaction: %d\n", rc);
 603                 lstcon_group_put(grp);
 604                 return rc;
 605         }
 606
 607         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 608
 609         lstcon_rpc_trans_destroy(trans);
 610
 611         lstcon_group_put(grp);
 612         /* -ref for session, it's destroyed,
 613          * status can't be rolled back, destroy group anyway */
 614         lstcon_group_put(grp);
 615
 616         return rc;
 617 }
 618
 619 int
 620 lstcon_group_clean(char *name, int args)
 621 {
 622         lstcon_group_t *grp = NULL;
 623         int rc;
 624
 625         rc = lstcon_group_find(name, &grp);
 626         if (rc != 0) {
 627                 CDEBUG(D_NET, "Can't find group %s\n", name);
 628                 return rc;
 629         }
 630
 631         if (grp->grp_ref > 2) {
 632                 /* referred by test */
 633                 CDEBUG(D_NET, "Group %s is busy\n", name);
 634                 lstcon_group_put(grp);
 635                 return -EBUSY;
 636         }
 637
 638         args = (LST_NODE_ACTIVE | LST_NODE_BUSY |
 639                 LST_NODE_DOWN | LST_NODE_UNKNOWN) & ~args;
 640
 641         lstcon_group_drain(grp, args);
 642
 643         lstcon_group_put(grp);
 644         /* release empty group */
 645         if (list_empty(&grp->grp_ndl_list))
 646                 lstcon_group_put(grp);
 647
 648         return 0;
 649 }
 650
 651 int
 652 lstcon_nodes_remove(char *name, int count,
 653                     lnet_process_id_t *ids_up, struct list_head *result_up)
 654 {
 655         lstcon_group_t *grp = NULL;
 656         int rc;
 657
 658         rc = lstcon_group_find(name, &grp);
 659         if (rc != 0) {
 660                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 661                 return rc;
 662         }
 663
 664         if (grp->grp_ref > 2) {
 665                 /* referred by test */
 666                 CDEBUG(D_NET, "Group %s is busy\n", name);
 667                 lstcon_group_put(grp);
 668                 return -EBUSY;
 669         }
 670
 671         rc = lstcon_group_nodes_remove(grp, count, ids_up, result_up);
 672
 673         lstcon_group_put(grp);
 674         /* release empty group */
 675         if (list_empty(&grp->grp_ndl_list))
 676                 lstcon_group_put(grp);
 677
 678         return rc;
 679 }
 680
 681 int
 682 lstcon_group_refresh(char *name, struct list_head *result_up)
 683 {
 684         lstcon_rpc_trans_t *trans;
 685         lstcon_group_t *grp;
 686         int rc;
 687
 688         rc = lstcon_group_find(name, &grp);
 689         if (rc != 0) {
 690                 CDEBUG(D_NET, "Can't find group: %s\n", name);
 691                 return rc;
 692         }
 693
 694         if (grp->grp_ref > 2) {
 695                 /* referred by test */
 696                 CDEBUG(D_NET, "Group %s is busy\n", name);
 697                 lstcon_group_put(grp);
 698                 return -EBUSY;
 699         }
 700
 701         /* re-invite all inactive nodes int the group */
 702         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
 703                                      &grp->grp_trans_list, LST_TRANS_SESNEW,
 704                                      grp, lstcon_sesrpc_condition, &trans);
 705         if (rc != 0) {
 706                 /* local error, return */
 707                 CDEBUG(D_NET, "Can't create transaction: %d\n", rc);
 708                 lstcon_group_put(grp);
 709                 return rc;
 710         }
 711
 712         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
 713
 714         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
 715
 716         lstcon_rpc_trans_destroy(trans);
 717         /* -ref for me */
 718         lstcon_group_put(grp);
 719
 720         return rc;
 721 }
 722
 723 int
 724 lstcon_group_list(int index, int len, char *name_up)
 725 {
 726         lstcon_group_t *grp;
 727
 728         LASSERT(index >= 0);
 729         LASSERT(name_up != NULL);
 730
 731         list_for_each_entry(grp, &console_session.ses_grp_list, grp_link) {
 732                 if (index-- == 0) {
 733                         return copy_to_user(name_up, grp->grp_name, len) ?
 734                                -EFAULT : 0;
 735                 }
 736         }
 737
 738         return -ENOENT;
 739 }
 740
 741 static int
 742 lstcon_nodes_getent(struct list_head *head, int *index_p,
 743                     int *count_p, lstcon_node_ent_t *dents_up)
 744 {
 745         lstcon_ndlink_t *ndl;
 746         lstcon_node_t *nd;
 747         int count = 0;
 748         int index = 0;
 749
 750         LASSERT(index_p != NULL && count_p != NULL);
 751         LASSERT(dents_up != NULL);
 752         LASSERT(*index_p >= 0);
 753         LASSERT(*count_p > 0);
 754
 755         list_for_each_entry(ndl, head, ndl_link) {
 756                 if (index++ < *index_p)
 757                         continue;
 758
 759                 if (count >= *count_p)
 760                         break;
 761
 762                 nd = ndl->ndl_node;
 763                 if (copy_to_user(&dents_up[count].nde_id,
 764                                      &nd->nd_id, sizeof(nd->nd_id)) ||
 765                     copy_to_user(&dents_up[count].nde_state,
 766                                      &nd->nd_state, sizeof(nd->nd_state)))
 767                         return -EFAULT;
 768
 769                 count++;
 770         }
 771
 772         if (index <= *index_p)
 773                 return -ENOENT;
 774
 775         *count_p = count;
 776         *index_p = index;
 777
 778         return 0;
 779 }
 780
 781 int
 782 lstcon_group_info(char *name, lstcon_ndlist_ent_t *gents_p,
 783                   int *index_p, int *count_p, lstcon_node_ent_t *dents_up)
 784 {
 785         lstcon_ndlist_ent_t *gentp;
 786         lstcon_group_t *grp;
 787         lstcon_ndlink_t *ndl;
 788         int rc;
 789
 790         rc = lstcon_group_find(name, &grp);
 791         if (rc != 0) {
 792                 CDEBUG(D_NET, "Can't find group %s\n", name);
 793                 return rc;
 794         }
 795
 796         if (dents_up) {
 797                 /* verbose query */
 798                 rc = lstcon_nodes_getent(&grp->grp_ndl_list,
 799                                          index_p, count_p, dents_up);
 800                 lstcon_group_put(grp);
 801
 802                 return rc;
 803         }
 804
 805         /* non-verbose query */
 806         LIBCFS_ALLOC(gentp, sizeof(lstcon_ndlist_ent_t));
 807         if (gentp == NULL) {
 808                 CERROR("Can't allocate ndlist_ent\n");
 809                 lstcon_group_put(grp);
 810
 811                 return -ENOMEM;
 812         }
 813
 814         list_for_each_entry(ndl, &grp->grp_ndl_list, ndl_link)
 815                 LST_NODE_STATE_COUNTER(ndl->ndl_node, gentp);
 816
 817         rc = copy_to_user(gents_p, gentp,
 818                               sizeof(lstcon_ndlist_ent_t)) ? -EFAULT : 0;
 819
 820         LIBCFS_FREE(gentp, sizeof(lstcon_ndlist_ent_t));
 821
 822         lstcon_group_put(grp);
 823
 824         return 0;
 825 }
 826
 827 static int
 828 lstcon_batch_find(const char *name, lstcon_batch_t **batpp)
 829 {
 830         lstcon_batch_t *bat;
 831
 832         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 833                 if (strncmp(bat->bat_name, name, LST_NAME_SIZE) == 0) {
 834                         *batpp = bat;
 835                         return 0;
 836                 }
 837         }
 838
 839         return -ENOENT;
 840 }
 841
 842 int
 843 lstcon_batch_add(char *name)
 844 {
 845         lstcon_batch_t *bat;
 846         int i;
 847         int rc;
 848
 849         rc = (lstcon_batch_find(name, &bat) == 0) ? -EEXIST : 0;
 850         if (rc != 0) {
 851                 CDEBUG(D_NET, "Batch %s already exists\n", name);
 852                 return rc;
 853         }
 854
 855         LIBCFS_ALLOC(bat, sizeof(lstcon_batch_t));
 856         if (bat == NULL) {
 857                 CERROR("Can't allocate descriptor for batch %s\n", name);
 858                 return -ENOMEM;
 859         }
 860
 861         LIBCFS_ALLOC(bat->bat_cli_hash,
 862                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 863         if (bat->bat_cli_hash == NULL) {
 864                 CERROR("Can't allocate hash for batch %s\n", name);
 865                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 866
 867                 return -ENOMEM;
 868         }
 869
 870         LIBCFS_ALLOC(bat->bat_srv_hash,
 871                      sizeof(struct list_head) * LST_NODE_HASHSIZE);
 872         if (bat->bat_srv_hash == NULL) {
 873                 CERROR("Can't allocate hash for batch %s\n", name);
 874                 LIBCFS_FREE(bat->bat_cli_hash, LST_NODE_HASHSIZE);
 875                 LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
 876
 877                 return -ENOMEM;
 878         }
 879
 880         strcpy(bat->bat_name, name);
 881         bat->bat_hdr.tsb_index = 0;
 882         bat->bat_hdr.tsb_id.bat_id = ++console_session.ses_id_cookie;
 883
 884         bat->bat_ntest = 0;
 885         bat->bat_state = LST_BATCH_IDLE;
 886
 887         INIT_LIST_HEAD(&bat->bat_cli_list);
 888         INIT_LIST_HEAD(&bat->bat_srv_list);
 889         INIT_LIST_HEAD(&bat->bat_test_list);
 890         INIT_LIST_HEAD(&bat->bat_trans_list);
 891
 892         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
 893                 INIT_LIST_HEAD(&bat->bat_cli_hash[i]);
 894                 INIT_LIST_HEAD(&bat->bat_srv_hash[i]);
 895         }
 896
 897         list_add_tail(&bat->bat_link, &console_session.ses_bat_list);
 898
 899         return rc;
 900 }
 901
 902 int
 903 lstcon_batch_list(int index, int len, char *name_up)
 904 {
 905         lstcon_batch_t *bat;
 906
 907         LASSERT(name_up != NULL);
 908         LASSERT(index >= 0);
 909
 910         list_for_each_entry(bat, &console_session.ses_bat_list, bat_link) {
 911                 if (index-- == 0) {
 912                         return copy_to_user(name_up, bat->bat_name, len) ?
 913                                -EFAULT : 0;
 914                 }
 915         }
 916
 917         return -ENOENT;
 918 }
 919
 920 int
 921 lstcon_batch_info(char *name, lstcon_test_batch_ent_t *ent_up, int server,
 922                   int testidx, int *index_p, int *ndent_p,
 923                   lstcon_node_ent_t *dents_up)
 924 {
 925         lstcon_test_batch_ent_t *entp;
 926         struct list_head *clilst;
 927         struct list_head *srvlst;
 928         lstcon_test_t *test = NULL;
 929         lstcon_batch_t *bat;
 930         lstcon_ndlink_t *ndl;
 931         int rc;
 932
 933         rc = lstcon_batch_find(name, &bat);
 934         if (rc != 0) {
 935                 CDEBUG(D_NET, "Can't find batch %s\n", name);
 936                 return -ENOENT;
 937         }
 938
 939         if (testidx > 0) {
 940                 /* query test, test index start from 1 */
 941                 list_for_each_entry(test, &bat->bat_test_list, tes_link) {
 942                         if (testidx-- == 1)
 943                                 break;
 944                 }
 945
 946                 if (testidx > 0) {
 947                         CDEBUG(D_NET, "Can't find specified test in batch\n");
 948                         return -ENOENT;
 949                 }
 950         }
 951
 952         clilst = (test == NULL) ? &bat->bat_cli_list :
 953                                   &test->tes_src_grp->grp_ndl_list;
 954         srvlst = (test == NULL) ? &bat->bat_srv_list :
 955                                   &test->tes_dst_grp->grp_ndl_list;
 956
 957         if (dents_up != NULL) {
 958                 rc = lstcon_nodes_getent((server ? srvlst : clilst),
 959                                          index_p, ndent_p, dents_up);
 960                 return rc;
 961         }
 962
 963         /* non-verbose query */
 964         LIBCFS_ALLOC(entp, sizeof(lstcon_test_batch_ent_t));
 965         if (entp == NULL)
 966                 return -ENOMEM;
 967
 968         if (test == NULL) {
 969                 entp->u.tbe_batch.bae_ntest = bat->bat_ntest;
 970                 entp->u.tbe_batch.bae_state = bat->bat_state;
 971
 972         } else {
 973
 974                 entp->u.tbe_test.tse_type   = test->tes_type;
 975                 entp->u.tbe_test.tse_loop   = test->tes_loop;
 976                 entp->u.tbe_test.tse_concur = test->tes_concur;
 977         }
 978
 979         list_for_each_entry(ndl, clilst, ndl_link)
 980                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_cli_nle);
 981
 982         list_for_each_entry(ndl, srvlst, ndl_link)
 983                 LST_NODE_STATE_COUNTER(ndl->ndl_node, &entp->tbe_srv_nle);
 984
 985         rc = copy_to_user(ent_up, entp,
 986                               sizeof(lstcon_test_batch_ent_t)) ? -EFAULT : 0;
 987
 988         LIBCFS_FREE(entp, sizeof(lstcon_test_batch_ent_t));
 989
 990         return rc;
 991 }
 992
 993 static int
 994 lstcon_batrpc_condition(int transop, lstcon_node_t *nd, void *arg)
 995 {
 996         switch (transop) {
 997         case LST_TRANS_TSBRUN:
 998                 if (nd->nd_state != LST_NODE_ACTIVE)
 999                         return -ENETDOWN;
1000                 break;
1001
1002         case LST_TRANS_TSBSTOP:
1003                 if (nd->nd_state != LST_NODE_ACTIVE)
1004                         return 0;
1005                 break;
1006
1007         case LST_TRANS_TSBCLIQRY:
1008         case LST_TRANS_TSBSRVQRY:
1009                 break;
1010         }
1011
1012         return 1;
1013 }
1014
1015 static int
1016 lstcon_batch_op(lstcon_batch_t *bat, int transop,
1017                 struct list_head *result_up)
1018 {
1019         lstcon_rpc_trans_t *trans;
1020         int rc;
1021
1022         rc = lstcon_rpc_trans_ndlist(&bat->bat_cli_list,
1023                                      &bat->bat_trans_list, transop,
1024                                      bat, lstcon_batrpc_condition, &trans);
1025         if (rc != 0) {
1026                 CERROR("Can't create transaction: %d\n", rc);
1027                 return rc;
1028         }
1029
1030         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1031
1032         rc = lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1033
1034         lstcon_rpc_trans_destroy(trans);
1035
1036         return rc;
1037 }
1038
1039 int
1040 lstcon_batch_run(char *name, int timeout, struct list_head *result_up)
1041 {
1042         lstcon_batch_t *bat;
1043         int rc;
1044
1045         if (lstcon_batch_find(name, &bat) != 0) {
1046                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1047                 return -ENOENT;
1048         }
1049
1050         bat->bat_arg = timeout;
1051
1052         rc = lstcon_batch_op(bat, LST_TRANS_TSBRUN, result_up);
1053
1054         /* mark batch as running if it's started in any node */
1055         if (lstcon_tsbop_stat_success(lstcon_trans_stat(), 0) != 0)
1056                 bat->bat_state = LST_BATCH_RUNNING;
1057
1058         return rc;
1059 }
1060
1061 int
1062 lstcon_batch_stop(char *name, int force, struct list_head *result_up)
1063 {
1064         lstcon_batch_t *bat;
1065         int rc;
1066
1067         if (lstcon_batch_find(name, &bat) != 0) {
1068                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1069                 return -ENOENT;
1070         }
1071
1072         bat->bat_arg = force;
1073
1074         rc = lstcon_batch_op(bat, LST_TRANS_TSBSTOP, result_up);
1075
1076         /* mark batch as stopped if all RPCs finished */
1077         if (lstcon_tsbop_stat_failure(lstcon_trans_stat(), 0) == 0)
1078                 bat->bat_state = LST_BATCH_IDLE;
1079
1080         return rc;
1081 }
1082
1083 static void
1084 lstcon_batch_destroy(lstcon_batch_t *bat)
1085 {
1086         lstcon_ndlink_t *ndl;
1087         lstcon_test_t *test;
1088         int i;
1089
1090         list_del(&bat->bat_link);
1091
1092         while (!list_empty(&bat->bat_test_list)) {
1093                 test = list_entry(bat->bat_test_list.next,
1094                                       lstcon_test_t, tes_link);
1095                 LASSERT(list_empty(&test->tes_trans_list));
1096
1097                 list_del(&test->tes_link);
1098
1099                 lstcon_group_put(test->tes_src_grp);
1100                 lstcon_group_put(test->tes_dst_grp);
1101
1102                 LIBCFS_FREE(test, offsetof(lstcon_test_t,
1103                                            tes_param[test->tes_paramlen]));
1104         }
1105
1106         LASSERT(list_empty(&bat->bat_trans_list));
1107
1108         while (!list_empty(&bat->bat_cli_list)) {
1109                 ndl = list_entry(bat->bat_cli_list.next,
1110                                      lstcon_ndlink_t, ndl_link);
1111                 list_del_init(&ndl->ndl_link);
1112
1113                 lstcon_ndlink_release(ndl);
1114         }
1115
1116         while (!list_empty(&bat->bat_srv_list)) {
1117                 ndl = list_entry(bat->bat_srv_list.next,
1118                                      lstcon_ndlink_t, ndl_link);
1119                 list_del_init(&ndl->ndl_link);
1120
1121                 lstcon_ndlink_release(ndl);
1122         }
1123
1124         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
1125                 LASSERT(list_empty(&bat->bat_cli_hash[i]));
1126                 LASSERT(list_empty(&bat->bat_srv_hash[i]));
1127         }
1128
1129         LIBCFS_FREE(bat->bat_cli_hash,
1130                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1131         LIBCFS_FREE(bat->bat_srv_hash,
1132                     sizeof(struct list_head) * LST_NODE_HASHSIZE);
1133         LIBCFS_FREE(bat, sizeof(lstcon_batch_t));
1134 }
1135
1136 static int
1137 lstcon_testrpc_condition(int transop, lstcon_node_t *nd, void *arg)
1138 {
1139         lstcon_test_t *test;
1140         lstcon_batch_t *batch;
1141         lstcon_ndlink_t *ndl;
1142         struct list_head *hash;
1143         struct list_head *head;
1144
1145         test = (lstcon_test_t *)arg;
1146         LASSERT(test != NULL);
1147
1148         batch = test->tes_batch;
1149         LASSERT(batch != NULL);
1150
1151         if (test->tes_oneside &&
1152             transop == LST_TRANS_TSBSRVADD)
1153                 return 0;
1154
1155         if (nd->nd_state != LST_NODE_ACTIVE)
1156                 return -ENETDOWN;
1157
1158         if (transop == LST_TRANS_TSBCLIADD) {
1159                 hash = batch->bat_cli_hash;
1160                 head = &batch->bat_cli_list;
1161
1162         } else {
1163                 LASSERT(transop == LST_TRANS_TSBSRVADD);
1164
1165                 hash = batch->bat_srv_hash;
1166                 head = &batch->bat_srv_list;
1167         }
1168
1169         LASSERT(nd->nd_id.nid != LNET_NID_ANY);
1170
1171         if (lstcon_ndlink_find(hash, nd->nd_id, &ndl, 1) != 0)
1172                 return -ENOMEM;
1173
1174         if (list_empty(&ndl->ndl_link))
1175                 list_add_tail(&ndl->ndl_link, head);
1176
1177         return 1;
1178 }
1179
1180 static int
1181 lstcon_test_nodes_add(lstcon_test_t *test, struct list_head *result_up)
1182 {
1183         lstcon_rpc_trans_t *trans;
1184         lstcon_group_t *grp;
1185         int transop;
1186         int rc;
1187
1188         LASSERT(test->tes_src_grp != NULL);
1189         LASSERT(test->tes_dst_grp != NULL);
1190
1191         transop = LST_TRANS_TSBSRVADD;
1192         grp  = test->tes_dst_grp;
1193 again:
1194         rc = lstcon_rpc_trans_ndlist(&grp->grp_ndl_list,
1195                                      &test->tes_trans_list, transop,
1196                                      test, lstcon_testrpc_condition, &trans);
1197         if (rc != 0) {
1198                 CERROR("Can't create transaction: %d\n", rc);
1199                 return rc;
1200         }
1201
1202         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1203
1204         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1205             lstcon_trans_stat()->trs_fwk_errno != 0) {
1206                 lstcon_rpc_trans_interpreter(trans, result_up, NULL);
1207
1208                 lstcon_rpc_trans_destroy(trans);
1209                 /* return if any error */
1210                 CDEBUG(D_NET, "Failed to add test %s, RPC error %d, framework error %d\n",
1211                        transop == LST_TRANS_TSBCLIADD ? "client" : "server",
1212                        lstcon_trans_stat()->trs_rpc_errno,
1213                        lstcon_trans_stat()->trs_fwk_errno);
1214
1215                 return rc;
1216         }
1217
1218         lstcon_rpc_trans_destroy(trans);
1219
1220         if (transop == LST_TRANS_TSBCLIADD)
1221                 return rc;
1222
1223         transop = LST_TRANS_TSBCLIADD;
1224         grp = test->tes_src_grp;
1225         test->tes_cliidx = 0;
1226
1227         /* requests to test clients */
1228         goto again;
1229 }
1230
1231 static int
1232 lstcon_verify_batch(const char *name, lstcon_batch_t **batch)
1233 {
1234         int rc;
1235
1236         rc = lstcon_batch_find(name, batch);
1237         if (rc != 0) {
1238                 CDEBUG(D_NET, "Can't find batch %s\n", name);
1239                 return rc;
1240         }
1241
1242         if ((*batch)->bat_state != LST_BATCH_IDLE) {
1243                 CDEBUG(D_NET, "Can't change running batch %s\n", name);
1244                 return -EINVAL;
1245         }
1246
1247         return 0;
1248 }
1249
1250 static int
1251 lstcon_verify_group(const char *name, lstcon_group_t **grp)
1252 {
1253         int rc;
1254         lstcon_ndlink_t *ndl;
1255
1256         rc = lstcon_group_find(name, grp);
1257         if (rc != 0) {
1258                 CDEBUG(D_NET, "can't find group %s\n", name);
1259                 return rc;
1260         }
1261
1262         list_for_each_entry(ndl, &(*grp)->grp_ndl_list, ndl_link) {
1263                 if (ndl->ndl_node->nd_state == LST_NODE_ACTIVE)
1264                         return 0;
1265         }
1266
1267         CDEBUG(D_NET, "Group %s has no ACTIVE nodes\n", name);
1268
1269         return -EINVAL;
1270 }
1271
1272 int
1273 lstcon_test_add(char *batch_name, int type, int loop,
1274                 int concur, int dist, int span,
1275                 char *src_name, char *dst_name,
1276                 void *param, int paramlen, int *retp,
1277                 struct list_head *result_up)
1278 {
1279         lstcon_test_t    *test   = NULL;
1280         int              rc;
1281         lstcon_group_t   *src_grp = NULL;
1282         lstcon_group_t   *dst_grp = NULL;
1283         lstcon_batch_t   *batch = NULL;
1284
1285         /*
1286          * verify that a batch of the given name exists, and the groups
1287          * that will be part of the batch exist and have at least one
1288          * active node
1289          */
1290         rc = lstcon_verify_batch(batch_name, &batch);
1291         if (rc != 0)
1292                 goto out;
1293
1294         rc = lstcon_verify_group(src_name, &src_grp);
1295         if (rc != 0)
1296                 goto out;
1297
1298         rc = lstcon_verify_group(dst_name, &dst_grp);
1299         if (rc != 0)
1300                 goto out;
1301
1302         if (dst_grp->grp_userland)
1303                 *retp = 1;
1304
1305         LIBCFS_ALLOC(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1306         if (!test) {
1307                 CERROR("Can't allocate test descriptor\n");
1308                 rc = -ENOMEM;
1309
1310                 goto out;
1311         }
1312
1313         test->tes_hdr.tsb_id    = batch->bat_hdr.tsb_id;
1314         test->tes_batch         = batch;
1315         test->tes_type          = type;
1316         test->tes_oneside       = 0; /* TODO */
1317         test->tes_loop          = loop;
1318         test->tes_concur        = concur;
1319         test->tes_stop_onerr    = 1; /* TODO */
1320         test->tes_span          = span;
1321         test->tes_dist          = dist;
1322         test->tes_cliidx        = 0; /* just used for creating RPC */
1323         test->tes_src_grp       = src_grp;
1324         test->tes_dst_grp       = dst_grp;
1325         INIT_LIST_HEAD(&test->tes_trans_list);
1326
1327         if (param != NULL) {
1328                 test->tes_paramlen = paramlen;
1329                 memcpy(&test->tes_param[0], param, paramlen);
1330         }
1331
1332         rc = lstcon_test_nodes_add(test, result_up);
1333
1334         if (rc != 0)
1335                 goto out;
1336
1337         if (lstcon_trans_stat()->trs_rpc_errno != 0 ||
1338             lstcon_trans_stat()->trs_fwk_errno != 0)
1339                 CDEBUG(D_NET, "Failed to add test %d to batch %s\n", type,
1340                        batch_name);
1341
1342         /* add to test list anyway, so user can check what's going on */
1343         list_add_tail(&test->tes_link, &batch->bat_test_list);
1344
1345         batch->bat_ntest++;
1346         test->tes_hdr.tsb_index = batch->bat_ntest;
1347
1348         /*  hold groups so nobody can change them */
1349         return rc;
1350 out:
1351         if (test != NULL)
1352                 LIBCFS_FREE(test, offsetof(lstcon_test_t, tes_param[paramlen]));
1353
1354         if (dst_grp != NULL)
1355                 lstcon_group_put(dst_grp);
1356
1357         if (src_grp != NULL)
1358                 lstcon_group_put(src_grp);
1359
1360         return rc;
1361 }
1362
1363 static int
1364 lstcon_test_find(lstcon_batch_t *batch, int idx, lstcon_test_t **testpp)
1365 {
1366         lstcon_test_t *test;
1367
1368         list_for_each_entry(test, &batch->bat_test_list, tes_link) {
1369                 if (idx == test->tes_hdr.tsb_index) {
1370                         *testpp = test;
1371                         return 0;
1372                 }
1373         }
1374
1375         return -ENOENT;
1376 }
1377
1378 static int
1379 lstcon_tsbrpc_readent(int transop, srpc_msg_t *msg,
1380                       lstcon_rpc_ent_t *ent_up)
1381 {
1382         srpc_batch_reply_t *rep = &msg->msg_body.bat_reply;
1383
1384         LASSERT(transop == LST_TRANS_TSBCLIQRY ||
1385                  transop == LST_TRANS_TSBSRVQRY);
1386
1387         /* positive errno, framework error code */
1388         if (copy_to_user(&ent_up->rpe_priv[0],
1389                              &rep->bar_active, sizeof(rep->bar_active)))
1390                 return -EFAULT;
1391
1392         return 0;
1393 }
1394
1395 int
1396 lstcon_test_batch_query(char *name, int testidx, int client,
1397                         int timeout, struct list_head *result_up)
1398 {
1399         lstcon_rpc_trans_t *trans;
1400         struct list_head *translist;
1401         struct list_head *ndlist;
1402         lstcon_tsb_hdr_t *hdr;
1403         lstcon_batch_t *batch;
1404         lstcon_test_t *test = NULL;
1405         int transop;
1406         int rc;
1407
1408         rc = lstcon_batch_find(name, &batch);
1409         if (rc != 0) {
1410                 CDEBUG(D_NET, "Can't find batch: %s\n", name);
1411                 return rc;
1412         }
1413
1414         if (testidx == 0) {
1415                 translist = &batch->bat_trans_list;
1416                 ndlist    = &batch->bat_cli_list;
1417                 hdr       = &batch->bat_hdr;
1418
1419         } else {
1420                 /* query specified test only */
1421                 rc = lstcon_test_find(batch, testidx, &test);
1422                 if (rc != 0) {
1423                         CDEBUG(D_NET, "Can't find test: %d\n", testidx);
1424                         return rc;
1425                 }
1426
1427                 translist = &test->tes_trans_list;
1428                 ndlist    = &test->tes_src_grp->grp_ndl_list;
1429                 hdr       = &test->tes_hdr;
1430         }
1431
1432         transop = client ? LST_TRANS_TSBCLIQRY : LST_TRANS_TSBSRVQRY;
1433
1434         rc = lstcon_rpc_trans_ndlist(ndlist, translist, transop, hdr,
1435                                      lstcon_batrpc_condition, &trans);
1436         if (rc != 0) {
1437                 CERROR("Can't create transaction: %d\n", rc);
1438                 return rc;
1439         }
1440
1441         lstcon_rpc_trans_postwait(trans, timeout);
1442
1443         if (testidx == 0 && /* query a batch, not a test */
1444             lstcon_rpc_stat_failure(lstcon_trans_stat(), 0) == 0 &&
1445             lstcon_tsbqry_stat_run(lstcon_trans_stat(), 0) == 0) {
1446                 /* all RPCs finished, and no active test */
1447                 batch->bat_state = LST_BATCH_IDLE;
1448         }
1449
1450         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1451                                           lstcon_tsbrpc_readent);
1452         lstcon_rpc_trans_destroy(trans);
1453
1454         return rc;
1455 }
1456
1457 static int
1458 lstcon_statrpc_readent(int transop, srpc_msg_t *msg,
1459                        lstcon_rpc_ent_t *ent_up)
1460 {
1461         srpc_stat_reply_t *rep = &msg->msg_body.stat_reply;
1462         sfw_counters_t *sfwk_stat;
1463         srpc_counters_t *srpc_stat;
1464         lnet_counters_t *lnet_stat;
1465
1466         if (rep->str_status != 0)
1467                 return 0;
1468
1469         sfwk_stat = (sfw_counters_t *)&ent_up->rpe_payload[0];
1470         srpc_stat = (srpc_counters_t *)((char *)sfwk_stat + sizeof(*sfwk_stat));
1471         lnet_stat = (lnet_counters_t *)((char *)srpc_stat + sizeof(*srpc_stat));
1472
1473         if (copy_to_user(sfwk_stat, &rep->str_fw, sizeof(*sfwk_stat)) ||
1474             copy_to_user(srpc_stat, &rep->str_rpc, sizeof(*srpc_stat)) ||
1475             copy_to_user(lnet_stat, &rep->str_lnet, sizeof(*lnet_stat)))
1476                 return -EFAULT;
1477
1478         return 0;
1479 }
1480
1481 static int
1482 lstcon_ndlist_stat(struct list_head *ndlist,
1483                    int timeout, struct list_head *result_up)
1484 {
1485         struct list_head head;
1486         lstcon_rpc_trans_t *trans;
1487         int rc;
1488
1489         INIT_LIST_HEAD(&head);
1490
1491         rc = lstcon_rpc_trans_ndlist(ndlist, &head,
1492                                      LST_TRANS_STATQRY, NULL, NULL, &trans);
1493         if (rc != 0) {
1494                 CERROR("Can't create transaction: %d\n", rc);
1495                 return rc;
1496         }
1497
1498         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1499
1500         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1501                                           lstcon_statrpc_readent);
1502         lstcon_rpc_trans_destroy(trans);
1503
1504         return rc;
1505 }
1506
1507 int
1508 lstcon_group_stat(char *grp_name, int timeout, struct list_head *result_up)
1509 {
1510         lstcon_group_t *grp;
1511         int rc;
1512
1513         rc = lstcon_group_find(grp_name, &grp);
1514         if (rc != 0) {
1515                 CDEBUG(D_NET, "Can't find group %s\n", grp_name);
1516                 return rc;
1517         }
1518
1519         rc = lstcon_ndlist_stat(&grp->grp_ndl_list, timeout, result_up);
1520
1521         lstcon_group_put(grp);
1522
1523         return rc;
1524 }
1525
1526 int
1527 lstcon_nodes_stat(int count, lnet_process_id_t *ids_up,
1528                   int timeout, struct list_head *result_up)
1529 {
1530         lstcon_ndlink_t *ndl;
1531         lstcon_group_t *tmp;
1532         lnet_process_id_t id;
1533         int i;
1534         int rc;
1535
1536         rc = lstcon_group_alloc(NULL, &tmp);
1537         if (rc != 0) {
1538                 CERROR("Out of memory\n");
1539                 return -ENOMEM;
1540         }
1541
1542         for (i = 0 ; i < count; i++) {
1543                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1544                         rc = -EFAULT;
1545                         break;
1546                 }
1547
1548                 /* add to tmp group */
1549                 rc = lstcon_group_ndlink_find(tmp, id, &ndl, 2);
1550                 if (rc != 0) {
1551                         CDEBUG((rc == -ENOMEM) ? D_ERROR : D_NET,
1552                                "Failed to find or create %s: %d\n",
1553                                libcfs_id2str(id), rc);
1554                         break;
1555                 }
1556         }
1557
1558         if (rc != 0) {
1559                 lstcon_group_put(tmp);
1560                 return rc;
1561         }
1562
1563         rc = lstcon_ndlist_stat(&tmp->grp_ndl_list, timeout, result_up);
1564
1565         lstcon_group_put(tmp);
1566
1567         return rc;
1568 }
1569
1570 static int
1571 lstcon_debug_ndlist(struct list_head *ndlist,
1572                     struct list_head *translist,
1573                     int timeout, struct list_head *result_up)
1574 {
1575         lstcon_rpc_trans_t *trans;
1576         int              rc;
1577
1578         rc = lstcon_rpc_trans_ndlist(ndlist, translist, LST_TRANS_SESQRY,
1579                                      NULL, lstcon_sesrpc_condition, &trans);
1580         if (rc != 0) {
1581                 CERROR("Can't create transaction: %d\n", rc);
1582                 return rc;
1583         }
1584
1585         lstcon_rpc_trans_postwait(trans, LST_VALIDATE_TIMEOUT(timeout));
1586
1587         rc = lstcon_rpc_trans_interpreter(trans, result_up,
1588                                           lstcon_sesrpc_readent);
1589         lstcon_rpc_trans_destroy(trans);
1590
1591         return rc;
1592 }
1593
1594 int
1595 lstcon_session_debug(int timeout, struct list_head *result_up)
1596 {
1597         return lstcon_debug_ndlist(&console_session.ses_ndl_list,
1598                                    NULL, timeout, result_up);
1599 }
1600
1601 int
1602 lstcon_batch_debug(int timeout, char *name,
1603                    int client, struct list_head *result_up)
1604 {
1605         lstcon_batch_t *bat;
1606         int rc;
1607
1608         rc = lstcon_batch_find(name, &bat);
1609         if (rc != 0)
1610                 return -ENOENT;
1611
1612         rc = lstcon_debug_ndlist(client ? &bat->bat_cli_list :
1613                                           &bat->bat_srv_list,
1614                                  NULL, timeout, result_up);
1615
1616         return rc;
1617 }
1618
1619 int
1620 lstcon_group_debug(int timeout, char *name,
1621                    struct list_head *result_up)
1622 {
1623         lstcon_group_t *grp;
1624         int rc;
1625
1626         rc = lstcon_group_find(name, &grp);
1627         if (rc != 0)
1628                 return -ENOENT;
1629
1630         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1631                                  timeout, result_up);
1632         lstcon_group_put(grp);
1633
1634         return rc;
1635 }
1636
1637 int
1638 lstcon_nodes_debug(int timeout,
1639                    int count, lnet_process_id_t *ids_up,
1640                    struct list_head *result_up)
1641 {
1642         lnet_process_id_t id;
1643         lstcon_ndlink_t *ndl;
1644         lstcon_group_t *grp;
1645         int i;
1646         int rc;
1647
1648         rc = lstcon_group_alloc(NULL, &grp);
1649         if (rc != 0) {
1650                 CDEBUG(D_NET, "Out of memory\n");
1651                 return rc;
1652         }
1653
1654         for (i = 0; i < count; i++) {
1655                 if (copy_from_user(&id, &ids_up[i], sizeof(id))) {
1656                         rc = -EFAULT;
1657                         break;
1658                 }
1659
1660                 /* node is added to tmp group */
1661                 rc = lstcon_group_ndlink_find(grp, id, &ndl, 1);
1662                 if (rc != 0) {
1663                         CERROR("Can't create node link\n");
1664                         break;
1665                 }
1666         }
1667
1668         if (rc != 0) {
1669                 lstcon_group_put(grp);
1670                 return rc;
1671         }
1672
1673         rc = lstcon_debug_ndlist(&grp->grp_ndl_list, NULL,
1674                                  timeout, result_up);
1675
1676         lstcon_group_put(grp);
1677
1678         return rc;
1679 }
1680
1681 int
1682 lstcon_session_match(lst_sid_t sid)
1683 {
1684         return (console_session.ses_id.ses_nid   == sid.ses_nid &&
1685                 console_session.ses_id.ses_stamp == sid.ses_stamp) ?  1 : 0;
1686 }
1687
1688 static void
1689 lstcon_new_session_id(lst_sid_t *sid)
1690 {
1691         lnet_process_id_t id;
1692
1693         LASSERT(console_session.ses_state == LST_SESSION_NONE);
1694
1695         LNetGetId(1, &id);
1696         sid->ses_nid   = id.nid;
1697         sid->ses_stamp = cfs_time_current();
1698 }
1699
1700 extern srpc_service_t lstcon_acceptor_service;
1701
1702 int
1703 lstcon_session_new(char *name, int key, unsigned feats,
1704                    int timeout, int force, lst_sid_t *sid_up)
1705 {
1706         int rc = 0;
1707         int i;
1708
1709         if (console_session.ses_state != LST_SESSION_NONE) {
1710                 /* session exists */
1711                 if (!force) {
1712                         CNETERR("Session %s already exists\n",
1713                                 console_session.ses_name);
1714                         return -EEXIST;
1715                 }
1716
1717                 rc = lstcon_session_end();
1718
1719                 /* lstcon_session_end() only return local error */
1720                 if  (rc != 0)
1721                         return rc;
1722         }
1723
1724         if ((feats & ~LST_FEATS_MASK) != 0) {
1725                 CNETERR("Unknown session features %x\n",
1726                         (feats & ~LST_FEATS_MASK));
1727                 return -EINVAL;
1728         }
1729
1730         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
1731                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
1732
1733         lstcon_new_session_id(&console_session.ses_id);
1734
1735         console_session.ses_key = key;
1736         console_session.ses_state = LST_SESSION_ACTIVE;
1737         console_session.ses_force = !!force;
1738         console_session.ses_features = feats;
1739         console_session.ses_feats_updated = 0;
1740         console_session.ses_timeout = (timeout <= 0) ?
1741                                       LST_CONSOLE_TIMEOUT : timeout;
1742         strcpy(console_session.ses_name, name);
1743
1744         rc = lstcon_batch_add(LST_DEFAULT_BATCH);
1745         if (rc != 0)
1746                 return rc;
1747
1748         rc = lstcon_rpc_pinger_start();
1749         if (rc != 0) {
1750                 lstcon_batch_t *bat = NULL;
1751
1752                 lstcon_batch_find(LST_DEFAULT_BATCH, &bat);
1753                 lstcon_batch_destroy(bat);
1754
1755                 return rc;
1756         }
1757
1758         if (copy_to_user(sid_up, &console_session.ses_id,
1759                              sizeof(lst_sid_t)) == 0)
1760                 return rc;
1761
1762         lstcon_session_end();
1763
1764         return -EFAULT;
1765 }
1766
1767 int
1768 lstcon_session_info(lst_sid_t *sid_up, int *key_up, unsigned *featp,
1769                     lstcon_ndlist_ent_t *ndinfo_up, char *name_up, int len)
1770 {
1771         lstcon_ndlist_ent_t *entp;
1772         lstcon_ndlink_t *ndl;
1773         int rc = 0;
1774
1775         if (console_session.ses_state != LST_SESSION_ACTIVE)
1776                 return -ESRCH;
1777
1778         LIBCFS_ALLOC(entp, sizeof(*entp));
1779         if (entp == NULL)
1780                 return -ENOMEM;
1781
1782         list_for_each_entry(ndl, &console_session.ses_ndl_list, ndl_link)
1783                 LST_NODE_STATE_COUNTER(ndl->ndl_node, entp);
1784
1785         if (copy_to_user(sid_up, &console_session.ses_id,
1786                              sizeof(lst_sid_t)) ||
1787             copy_to_user(key_up, &console_session.ses_key,
1788                              sizeof(*key_up)) ||
1789             copy_to_user(featp, &console_session.ses_features,
1790                              sizeof(*featp)) ||
1791             copy_to_user(ndinfo_up, entp, sizeof(*entp)) ||
1792             copy_to_user(name_up, console_session.ses_name, len))
1793                 rc = -EFAULT;
1794
1795         LIBCFS_FREE(entp, sizeof(*entp));
1796
1797         return rc;
1798 }
1799
1800 int
1801 lstcon_session_end(void)
1802 {
1803         lstcon_rpc_trans_t *trans;
1804         lstcon_group_t *grp;
1805         lstcon_batch_t *bat;
1806         int rc = 0;
1807
1808         LASSERT(console_session.ses_state == LST_SESSION_ACTIVE);
1809
1810         rc = lstcon_rpc_trans_ndlist(&console_session.ses_ndl_list,
1811                                      NULL, LST_TRANS_SESEND, NULL,
1812                                      lstcon_sesrpc_condition, &trans);
1813         if (rc != 0) {
1814                 CERROR("Can't create transaction: %d\n", rc);
1815                 return rc;
1816         }
1817
1818         console_session.ses_shutdown = 1;
1819
1820         lstcon_rpc_pinger_stop();
1821
1822         lstcon_rpc_trans_postwait(trans, LST_TRANS_TIMEOUT);
1823
1824         lstcon_rpc_trans_destroy(trans);
1825         /* User can do nothing even rpc failed, so go on */
1826
1827         /* waiting for orphan rpcs to die */
1828         lstcon_rpc_cleanup_wait();
1829
1830         console_session.ses_id    = LST_INVALID_SID;
1831         console_session.ses_state = LST_SESSION_NONE;
1832         console_session.ses_key   = 0;
1833         console_session.ses_force = 0;
1834         console_session.ses_feats_updated = 0;
1835
1836         /* destroy all batches */
1837         while (!list_empty(&console_session.ses_bat_list)) {
1838                 bat = list_entry(console_session.ses_bat_list.next,
1839                                      lstcon_batch_t, bat_link);
1840
1841                 lstcon_batch_destroy(bat);
1842         }
1843
1844         /* destroy all groups */
1845         while (!list_empty(&console_session.ses_grp_list)) {
1846                 grp = list_entry(console_session.ses_grp_list.next,
1847                                      lstcon_group_t, grp_link);
1848                 LASSERT(grp->grp_ref == 1);
1849
1850                 lstcon_group_put(grp);
1851         }
1852
1853         /* all nodes should be released */
1854         LASSERT(list_empty(&console_session.ses_ndl_list));
1855
1856         console_session.ses_shutdown = 0;
1857         console_session.ses_expired  = 0;
1858
1859         return rc;
1860 }
1861
1862 int
1863 lstcon_session_feats_check(unsigned feats)
1864 {
1865         int rc = 0;
1866
1867         if ((feats & ~LST_FEATS_MASK) != 0) {
1868                 CERROR("Can't support these features: %x\n",
1869                        (feats & ~LST_FEATS_MASK));
1870                 return -EPROTO;
1871         }
1872
1873         spin_lock(&console_session.ses_rpc_lock);
1874
1875         if (!console_session.ses_feats_updated) {
1876                 console_session.ses_feats_updated = 1;
1877                 console_session.ses_features = feats;
1878         }
1879
1880         if (console_session.ses_features != feats)
1881                 rc = -EPROTO;
1882
1883         spin_unlock(&console_session.ses_rpc_lock);
1884
1885         if (rc != 0) {
1886                 CERROR("remote features %x do not match with session features %x of console\n",
1887                        feats, console_session.ses_features);
1888         }
1889
1890         return rc;
1891 }
1892
1893 static int
1894 lstcon_acceptor_handle(srpc_server_rpc_t *rpc)
1895 {
1896         srpc_msg_t *rep  = &rpc->srpc_replymsg;
1897         srpc_msg_t *req  = &rpc->srpc_reqstbuf->buf_msg;
1898         srpc_join_reqst_t *jreq = &req->msg_body.join_reqst;
1899         srpc_join_reply_t *jrep = &rep->msg_body.join_reply;
1900         lstcon_group_t *grp  = NULL;
1901         lstcon_ndlink_t *ndl;
1902         int rc   = 0;
1903
1904         sfw_unpack_message(req);
1905
1906         mutex_lock(&console_session.ses_mutex);
1907
1908         jrep->join_sid = console_session.ses_id;
1909
1910         if (console_session.ses_id.ses_nid == LNET_NID_ANY) {
1911                 jrep->join_status = ESRCH;
1912                 goto out;
1913         }
1914
1915         if (lstcon_session_feats_check(req->msg_ses_feats) != 0) {
1916                 jrep->join_status = EPROTO;
1917                 goto out;
1918         }
1919
1920         if (jreq->join_sid.ses_nid != LNET_NID_ANY &&
1921              !lstcon_session_match(jreq->join_sid)) {
1922                 jrep->join_status = EBUSY;
1923                 goto out;
1924         }
1925
1926         if (lstcon_group_find(jreq->join_group, &grp) != 0) {
1927                 rc = lstcon_group_alloc(jreq->join_group, &grp);
1928                 if (rc != 0) {
1929                         CERROR("Out of memory\n");
1930                         goto out;
1931                 }
1932
1933                 list_add_tail(&grp->grp_link,
1934                                   &console_session.ses_grp_list);
1935                 lstcon_group_addref(grp);
1936         }
1937
1938         if (grp->grp_ref > 2) {
1939                 /* Group in using */
1940                 jrep->join_status = EBUSY;
1941                 goto out;
1942         }
1943
1944         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 0);
1945         if (rc == 0) {
1946                 jrep->join_status = EEXIST;
1947                 goto out;
1948         }
1949
1950         rc = lstcon_group_ndlink_find(grp, rpc->srpc_peer, &ndl, 1);
1951         if (rc != 0) {
1952                 CERROR("Out of memory\n");
1953                 goto out;
1954         }
1955
1956         ndl->ndl_node->nd_state   = LST_NODE_ACTIVE;
1957         ndl->ndl_node->nd_timeout = console_session.ses_timeout;
1958
1959         if (grp->grp_userland == 0)
1960                 grp->grp_userland = 1;
1961
1962         strcpy(jrep->join_session, console_session.ses_name);
1963         jrep->join_timeout = console_session.ses_timeout;
1964         jrep->join_status  = 0;
1965
1966 out:
1967         rep->msg_ses_feats = console_session.ses_features;
1968         if (grp != NULL)
1969                 lstcon_group_put(grp);
1970
1971         mutex_unlock(&console_session.ses_mutex);
1972
1973         return rc;
1974 }
1975
1976 srpc_service_t lstcon_acceptor_service;
1977 static void lstcon_init_acceptor_service(void)
1978 {
1979         /* initialize selftest console acceptor service table */
1980         lstcon_acceptor_service.sv_name     = "join session";
1981         lstcon_acceptor_service.sv_handler  = lstcon_acceptor_handle;
1982         lstcon_acceptor_service.sv_id       = SRPC_SERVICE_JOIN;
1983         lstcon_acceptor_service.sv_wi_total = SFW_FRWK_WI_MAX;
1984 }
1985
1986 extern int lstcon_ioctl_entry(unsigned int cmd, struct libcfs_ioctl_data *data);
1987
1988 static DECLARE_IOCTL_HANDLER(lstcon_ioctl_handler, lstcon_ioctl_entry);
1989
1990 /* initialize console */
1991 int
1992 lstcon_console_init(void)
1993 {
1994         int i;
1995         int rc;
1996
1997         memset(&console_session, 0, sizeof(lstcon_session_t));
1998
1999         console_session.ses_id            = LST_INVALID_SID;
2000         console_session.ses_state         = LST_SESSION_NONE;
2001         console_session.ses_timeout       = 0;
2002         console_session.ses_force         = 0;
2003         console_session.ses_expired       = 0;
2004         console_session.ses_feats_updated = 0;
2005         console_session.ses_features      = LST_FEATS_MASK;
2006         console_session.ses_laststamp     = ktime_get_real_seconds();
2007
2008         mutex_init(&console_session.ses_mutex);
2009
2010         INIT_LIST_HEAD(&console_session.ses_ndl_list);
2011         INIT_LIST_HEAD(&console_session.ses_grp_list);
2012         INIT_LIST_HEAD(&console_session.ses_bat_list);
2013         INIT_LIST_HEAD(&console_session.ses_trans_list);
2014
2015         LIBCFS_ALLOC(console_session.ses_ndl_hash,
2016                      sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2017         if (console_session.ses_ndl_hash == NULL)
2018                 return -ENOMEM;
2019
2020         for (i = 0; i < LST_GLOBAL_HASHSIZE; i++)
2021                 INIT_LIST_HEAD(&console_session.ses_ndl_hash[i]);
2022
2023         /* initialize acceptor service table */
2024         lstcon_init_acceptor_service();
2025
2026         rc = srpc_add_service(&lstcon_acceptor_service);
2027         LASSERT(rc != -EBUSY);
2028         if (rc != 0) {
2029                 LIBCFS_FREE(console_session.ses_ndl_hash,
2030                             sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2031                 return rc;
2032         }
2033
2034         rc = srpc_service_add_buffers(&lstcon_acceptor_service,
2035                                       lstcon_acceptor_service.sv_wi_total);
2036         if (rc != 0) {
2037                 rc = -ENOMEM;
2038                 goto out;
2039         }
2040
2041         rc = libcfs_register_ioctl(&lstcon_ioctl_handler);
2042
2043         if (rc == 0) {
2044                 lstcon_rpc_module_init();
2045                 return 0;
2046         }
2047
2048 out:
2049         srpc_shutdown_service(&lstcon_acceptor_service);
2050         srpc_remove_service(&lstcon_acceptor_service);
2051
2052         LIBCFS_FREE(console_session.ses_ndl_hash,
2053                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2054
2055         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2056
2057         return rc;
2058 }
2059
2060 int
2061 lstcon_console_fini(void)
2062 {
2063         int i;
2064
2065         libcfs_deregister_ioctl(&lstcon_ioctl_handler);
2066
2067         mutex_lock(&console_session.ses_mutex);
2068
2069         srpc_shutdown_service(&lstcon_acceptor_service);
2070         srpc_remove_service(&lstcon_acceptor_service);
2071
2072         if (console_session.ses_state != LST_SESSION_NONE)
2073                 lstcon_session_end();
2074
2075         lstcon_rpc_module_fini();
2076
2077         mutex_unlock(&console_session.ses_mutex);
2078
2079         LASSERT(list_empty(&console_session.ses_ndl_list));
2080         LASSERT(list_empty(&console_session.ses_grp_list));
2081         LASSERT(list_empty(&console_session.ses_bat_list));
2082         LASSERT(list_empty(&console_session.ses_trans_list));
2083
2084         for (i = 0; i < LST_NODE_HASHSIZE; i++) {
2085                 LASSERT(list_empty(&console_session.ses_ndl_hash[i]));
2086         }
2087
2088         LIBCFS_FREE(console_session.ses_ndl_hash,
2089                     sizeof(struct list_head) * LST_GLOBAL_HASHSIZE);
2090
2091         srpc_wait_service_shutdown(&lstcon_acceptor_service);
2092
2093         return 0;
2094 }