Merge pull request #43942 from rhcs-dashboard/host-drain-frontend

mgr/dashboard: Implement drain host functionality in dashboard

Reviewed-by: Alfonso Martínez <almartin@redhat.com>
Reviewed-by: Avan Thakkar <athakkar@redhat.com>
Reviewed-by: Ernesto Puerta <epuertat@redhat.com>
Reviewed-by: Nizamudeen A <nia@redhat.com>

src/pybind/mgr/dashboard/ci/cephadm/ceph_cluster.yml

@@ -1,5 +1,5 @@
 parameters:
- nodes: 3
+ nodes: 4
  pool: ceph-dashboard
  network: ceph-dashboard
  domain: cephlab.com

src/pybind/mgr/dashboard/controllers/host.py

@@ -419,7 +419,8 @@ class Host(RESTController):
     @raise_if_no_orchestrator([OrchFeature.HOST_LABEL_ADD,
                                OrchFeature.HOST_LABEL_REMOVE,
                                OrchFeature.HOST_MAINTENANCE_ENTER,
-                               OrchFeature.HOST_MAINTENANCE_EXIT])
+                               OrchFeature.HOST_MAINTENANCE_EXIT,
+                               OrchFeature.HOST_DRAIN])
     @handle_orchestrator_error('host')
     @EndpointDoc('',
                  parameters={
@@ -427,13 +428,14 @@ class Host(RESTController):
                      'update_labels': (bool, 'Update Labels'),
                      'labels': ([str], 'Host Labels'),
                      'maintenance': (bool, 'Enter/Exit Maintenance'),
-                     'force': (bool, 'Force Enter Maintenance')
+                     'force': (bool, 'Force Enter Maintenance'),
+                     'drain': (bool, 'Drain Host')
                  },
                  responses={200: None, 204: None})
     @RESTController.MethodMap(version=APIVersion.EXPERIMENTAL)
     def set(self, hostname: str, update_labels: bool = False,
             labels: List[str] = None, maintenance: bool = False,
-            force: bool = False):
+            force: bool = False, drain: bool = False):
         """
         Update the specified host.
         Note, this is only supported when Ceph Orchestrator is enabled.
@@ -442,6 +444,7 @@ class Host(RESTController):
         :param labels: List of labels.
         :param maintenance: Enter/Exit maintenance mode.
         :param force: Force enter maintenance mode.
+        :param drain: Drain host
         """
         orch = OrchClient.instance()
         host = get_host(hostname)
@@ -454,6 +457,9 @@ class Host(RESTController):
             if status == 'maintenance':
                 orch.hosts.exit_maintenance(hostname)
 
+        if drain:
+            orch.hosts.drain(hostname)
+
         if update_labels:
             # only allow List[str] type for labels
             if not isinstance(labels, list):

src/pybind/mgr/dashboard/frontend/cypress/integration/cluster/hosts.po.ts

@@ -80,7 +80,7 @@ export class HostsPageHelper extends PageHelper {
     });
   }
 
-  delete(hostname: string) {
+  remove(hostname: string) {
     super.delete(hostname, this.columnIndex.hostname, 'hosts');
   }
 
@@ -173,4 +173,17 @@ export class HostsPageHelper extends PageHelper {
         });
     }
   }
+
+  @PageHelper.restrictTo(pages.index.url)
+  drain(hostname: string) {
+    this.getTableCell(this.columnIndex.hostname, hostname).click();
+    this.clickActionButton('start-drain');
+    this.checkLabelExists(hostname, ['_no_schedule'], true);
+
+    this.clickTab('cd-host-details', hostname, 'Daemons');
+    cy.get('cd-host-details').within(() => {
+      cy.wait(10000);
+      this.expectTableCount('total', 0);
+    });
+  }
 }

src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/01-hosts.e2e-spec.ts

@@ -21,21 +21,12 @@ describe('Hosts page', () => {
       hosts.add(hostname, true);
     });
 
-    it('should drain and delete a host and then add it back', function () {
+    it('should drain and remove a host and then add it back', function () {
       const hostname = Cypress._.last(this.hosts)['name'];
 
       // should drain the host first before deleting
-      hosts.editLabels(hostname, ['_no_schedule'], true);
-      hosts.clickTab('cd-host-details', hostname, 'Daemons');
-      cy.get('cd-host-details').within(() => {
-        // draining will take some time to complete.
-        // since we don't know how many daemons will be
-        // running in this host in future putting the wait
-        // to 15s
-        cy.wait(15000);
-        hosts.getTableCount('total').should('be.eq', 0);
-      });
-      hosts.delete(hostname);
+      hosts.drain(hostname);
+      hosts.remove(hostname);
 
       // add it back
       hosts.navigateTo('add');

src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/02-create-cluster-add-host.e2e-spec.ts

@@ -10,7 +10,7 @@ describe('Create cluster add host page', () => {
     'ceph-node-00.cephlab.com',
     'ceph-node-01.cephlab.com',
     'ceph-node-02.cephlab.com',
-    'ceph-node-[01-02].cephlab.com'
+    'ceph-node-[01-03].cephlab.com'
   ];
   const addHost = (hostname: string, exist?: boolean, pattern?: boolean, labels: string[] = []) => {
     cy.get('.btn.btn-accent').first().click({ force: true });
@@ -38,13 +38,13 @@ describe('Create cluster add host page', () => {
 
     addHost(hostnames[1], false);
     addHost(hostnames[2], false);
-    createClusterHostPage.delete(hostnames[1]);
-    createClusterHostPage.delete(hostnames[2]);
+    createClusterHostPage.remove(hostnames[1]);
+    createClusterHostPage.remove(hostnames[2]);
     addHost(hostnames[3], false, true);
   });
 
-  it('should delete a host', () => {
-    createClusterHostPage.delete(hostnames[1]);
+  it('should remove a host', () => {
+    createClusterHostPage.remove(hostnames[1]);
   });
 
   it('should add a host with some predefined labels and verify it', () => {

src/pybind/mgr/dashboard/frontend/cypress/integration/orchestrator/workflow/06-cluster-check.e2e-spec.ts

@@ -27,7 +27,8 @@ describe('when cluster creation is completed', () => {
     const hostnames = [
       'ceph-node-00.cephlab.com',
       'ceph-node-01.cephlab.com',
-      'ceph-node-02.cephlab.com'
+      'ceph-node-02.cephlab.com',
+      'ceph-node-03.cephlab.com'
     ];
 
     beforeEach(() => {
@@ -55,14 +56,22 @@ describe('when cluster creation is completed', () => {
     });
 
     it('should check if rgw service is running', () => {
-      hosts.clickTab('cd-host-details', hostnames[1], 'Daemons');
+      hosts.clickTab('cd-host-details', hostnames[3], 'Daemons');
       cy.get('cd-host-details').within(() => {
         services.checkServiceStatus('rgw');
       });
     });
 
     it('should force maintenance and exit', { retries: 1 }, () => {
-      hosts.maintenance(hostnames[1], true, true);
+      hosts.maintenance(hostnames[3], true, true);
+    });
+
+    it('should drain, remove and add the host back', () => {
+      hosts.drain(hostnames[1]);
+      hosts.remove(hostnames[1]);
+      hosts.navigateTo('add');
+      hosts.add(hostnames[1]);
+      hosts.checkExist(hostnames[1], true);
     });
   });
 

src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.spec.ts

@@ -294,7 +294,8 @@ describe('HostsComponent', () => {
         OrchestratorFeature.HOST_ADD,
         OrchestratorFeature.HOST_LABEL_ADD,
         OrchestratorFeature.HOST_REMOVE,
-        OrchestratorFeature.HOST_LABEL_REMOVE
+        OrchestratorFeature.HOST_LABEL_REMOVE,
+        OrchestratorFeature.HOST_DRAIN
       ];
       await testTableActions(true, features, tests);
     });

src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/hosts/hosts.component.ts

@@ -84,7 +84,8 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
   modalRef: NgbModalRef;
   isExecuting = false;
   errorMessage: string;
-  enableButton: boolean;
+  enableMaintenanceBtn: boolean;
+  enableDrainBtn: boolean;
   bsModalRef: NgbModalRef;
 
   icons = Icons;
@@ -101,7 +102,8 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
     maintenance: [
       OrchestratorFeature.HOST_MAINTENANCE_ENTER,
       OrchestratorFeature.HOST_MAINTENANCE_EXIT
-    ]
+    ],
+    drain: [OrchestratorFeature.HOST_DRAIN]
   };
 
   constructor(
@@ -135,6 +137,24 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
         click: () => this.editAction(),
         disable: (selection: CdTableSelection) => this.getDisable('edit', selection)
       },
+      {
+        name: this.actionLabels.START_DRAIN,
+        permission: 'update',
+        icon: Icons.exit,
+        click: () => this.hostDrain(),
+        disable: (selection: CdTableSelection) =>
+          this.getDisable('drain', selection) || !this.enableDrainBtn,
+        visible: () => !this.showGeneralActionsOnly && this.enableDrainBtn
+      },
+      {
+        name: this.actionLabels.STOP_DRAIN,
+        permission: 'update',
+        icon: Icons.exit,
+        click: () => this.hostDrain(true),
+        disable: (selection: CdTableSelection) =>
+          this.getDisable('drain', selection) || this.enableDrainBtn,
+        visible: () => !this.showGeneralActionsOnly && !this.enableDrainBtn
+      },
       {
         name: this.actionLabels.REMOVE,
         permission: 'delete',
@@ -148,8 +168,10 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
         icon: Icons.enter,
         click: () => this.hostMaintenance(),
         disable: (selection: CdTableSelection) =>
-          this.getDisable('maintenance', selection) || this.isExecuting || this.enableButton,
-        visible: () => !this.showGeneralActionsOnly
+          this.getDisable('maintenance', selection) ||
+          this.isExecuting ||
+          this.enableMaintenanceBtn,
+        visible: () => !this.showGeneralActionsOnly && !this.enableMaintenanceBtn
       },
       {
         name: this.actionLabels.EXIT_MAINTENANCE,
@@ -157,8 +179,10 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
         icon: Icons.exit,
         click: () => this.hostMaintenance(),
         disable: (selection: CdTableSelection) =>
-          this.getDisable('maintenance', selection) || this.isExecuting || !this.enableButton,
-        visible: () => !this.showGeneralActionsOnly
+          this.getDisable('maintenance', selection) ||
+          this.isExecuting ||
+          !this.enableMaintenanceBtn,
+        visible: () => !this.showGeneralActionsOnly && this.enableMaintenanceBtn
       }
     ];
   }
@@ -252,10 +276,15 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
 
   updateSelection(selection: CdTableSelection) {
     this.selection = selection;
-    this.enableButton = false;
+    this.enableMaintenanceBtn = false;
+    this.enableDrainBtn = false;
     if (this.selection.hasSelection) {
       if (this.selection.first().status === 'maintenance') {
-        this.enableButton = true;
+        this.enableMaintenanceBtn = true;
+      }
+
+      if (!this.selection.first().labels.includes('_no_schedule')) {
+        this.enableDrainBtn = true;
       }
     }
   }
@@ -361,11 +390,39 @@ export class HostsComponent extends ListWithDetails implements OnDestroy, OnInit
     }
   }
 
+  hostDrain(stop = false) {
+    const host = this.selection.first();
+    if (stop) {
+      const index = host['labels'].indexOf('_no_schedule', 0);
+      host['labels'].splice(index, 1);
+      this.hostService.update(host['hostname'], true, host['labels']).subscribe(() => {
+        this.notificationService.show(
+          NotificationType.info,
+          $localize`"${host['hostname']}" stopped draining`
+        );
+        this.table.refreshBtn();
+      });
+    } else {
+      this.hostService.update(host['hostname'], false, [], false, false, true).subscribe(() => {
+        this.notificationService.show(
+          NotificationType.info,
+          $localize`"${host['hostname']}" started draining`
+        );
+        this.table.refreshBtn();
+      });
+    }
+  }
+
   getDisable(
-    action: 'add' | 'edit' | 'remove' | 'maintenance',
+    action: 'add' | 'edit' | 'remove' | 'maintenance' | 'drain',
     selection: CdTableSelection
   ): boolean | string {
-    if (action === 'remove' || action === 'edit' || action === 'maintenance') {
+    if (
+      action === 'remove' ||
+      action === 'edit' ||
+      action === 'maintenance' ||
+      action === 'drain'
+    ) {
       if (!selection?.hasSingleSelection) {
         return true;
       }

src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.spec.ts

@@ -44,14 +44,28 @@ describe('HostService', () => {
   });
 
   it('should update host', fakeAsync(() => {
-    service.update('mon0', true, ['foo', 'bar']).subscribe();
+    service.update('mon0', true, ['foo', 'bar'], true, false).subscribe();
     const req = httpTesting.expectOne('api/host/mon0');
     expect(req.request.method).toBe('PUT');
     expect(req.request.body).toEqual({
       force: false,
       labels: ['foo', 'bar'],
+      maintenance: true,
+      update_labels: true,
+      drain: false
+    });
+  }));
+
+  it('should test host drain call', fakeAsync(() => {
+    service.update('host0', false, null, false, false, true).subscribe();
+    const req = httpTesting.expectOne('api/host/host0');
+    expect(req.request.method).toBe('PUT');
+    expect(req.request.body).toEqual({
+      force: false,
+      labels: null,
       maintenance: false,
-      update_labels: true
+      update_labels: false,
+      drain: true
     });
   }));
 

src/pybind/mgr/dashboard/frontend/src/app/shared/api/host.service.ts

@@ -69,7 +69,8 @@ export class HostService extends ApiClient {
     updateLabels = false,
     labels: string[] = [],
     maintenance = false,
-    force = false
+    force = false,
+    drain = false
   ) {
     return this.http.put(
       `${this.baseURL}/${hostname}`,
@@ -77,7 +78,8 @@ export class HostService extends ApiClient {
         update_labels: updateLabels,
         labels: labels,
         maintenance: maintenance,
-        force: force
+        force: force,
+        drain: drain
       },
       { headers: { Accept: this.getVersionHeaderValue(0, 1) } }
     );

src/pybind/mgr/dashboard/frontend/src/app/shared/constants/app.constants.ts

@@ -118,6 +118,8 @@ export class ActionLabelsI18n {
   FLAGS: string;
   ENTER_MAINTENANCE: string;
   EXIT_MAINTENANCE: string;
+  START_DRAIN: string;
+  STOP_DRAIN: string;
 
   constructor() {
     /* Create a new item */
@@ -171,6 +173,8 @@ export class ActionLabelsI18n {
     this.FLAGS = $localize`Flags`;
     this.ENTER_MAINTENANCE = $localize`Enter Maintenance`;
     this.EXIT_MAINTENANCE = $localize`Exit Maintenance`;
+    this.START_DRAIN = $localize`Start Drain`;
+    this.STOP_DRAIN = $localize`Stop Drain`;
 
     /* Prometheus wording */
     this.RECREATE = $localize`Recreate`;

src/pybind/mgr/dashboard/frontend/src/app/shared/models/orchestrator.enum.ts

@@ -7,6 +7,7 @@ export enum OrchestratorFeature {
   HOST_MAINTENANCE_ENTER = 'enter_host_maintenance',
   HOST_MAINTENANCE_EXIT = 'exit_host_maintenance',
   HOST_FACTS = 'get_facts',
+  HOST_DRAIN = 'drain_host',
 
   SERVICE_LIST = 'describe_service',
   SERVICE_CREATE = 'apply',

src/pybind/mgr/dashboard/openapi.yaml

@@ -3492,7 +3492,7 @@ paths:
         \ name of the host to be processed.\n        :param update_labels: To update\
         \ the labels.\n        :param labels: List of labels.\n        :param maintenance:\
         \ Enter/Exit maintenance mode.\n        :param force: Force enter maintenance\
-        \ mode.\n        "
+        \ mode.\n        :param drain: Drain host\n        "
       parameters:
       - description: Hostname
         in: path
@@ -3505,6 +3505,10 @@ paths:
           application/json:
             schema:
               properties:
+                drain:
+                  default: false
+                  description: Drain Host
+                  type: boolean
                 force:
                   default: false
                   description: Force Enter Maintenance

src/pybind/mgr/dashboard/services/orchestrator.py

@@ -82,6 +82,10 @@ class HostManger(ResourceManager):
     def remove_label(self, host: str, label: str) -> OrchResult[str]:
         return self.api.remove_host_label(host, label)
 
+    @wait_api_result
+    def drain(self, hostname: str):
+        return self.api.drain_host(hostname)
+
 
 class InventoryManager(ResourceManager):
     @wait_api_result
@@ -198,6 +202,7 @@ class OrchFeature(object):
     HOST_LABEL_REMOVE = 'remove_host_label'
     HOST_MAINTENANCE_ENTER = 'enter_host_maintenance'
     HOST_MAINTENANCE_EXIT = 'exit_host_maintenance'
+    HOST_DRAIN = 'drain_host'
 
     SERVICE_LIST = 'describe_service'
     SERVICE_CREATE = 'apply'

src/pybind/mgr/dashboard/tests/test_host.py

@@ -332,6 +332,24 @@ class HostControllerTest(ControllerTestCase):
             self._get(inventory_url)
             self.assertStatus(503)
 
+    def test_host_drain(self):
+        mgr.list_servers.return_value = []
+        orch_hosts = [
+            HostSpec('node0')
+        ]
+        with patch_orch(True, hosts=orch_hosts):
+            self._put('{}/node0'.format(self.URL_HOST), {'drain': True},
+                      version=APIVersion(0, 1))
+            self.assertStatus(200)
+            self.assertHeader('Content-Type',
+                              'application/vnd.ceph.api.v0.1+json')
+
+        # maintenance without orchestrator service
+        with patch_orch(False):
+            self._put('{}/node0'.format(self.URL_HOST), {'drain': True},
+                      version=APIVersion(0, 1))
+            self.assertStatus(503)
+
 
 class HostUiControllerTest(ControllerTestCase):
     URL_HOST = '/ui-api/host'