diff --git a/doc/mgr/dashboard.rst b/doc/mgr/dashboard.rst index cd0b92b5ed2..15e793c2e21 100644 --- a/doc/mgr/dashboard.rst +++ b/doc/mgr/dashboard.rst @@ -392,6 +392,81 @@ To enable SSO:: $ ceph dashboard sso enable saml2 +Enabling Prometheus alerting +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Using Prometheus for monitoring, you have to define `alerting rules +`_. +To manage them you need to use the `Alertmanager +`_. +If you are not using the Alertmanager yet, please `install it +`_ as it's mandatory in +order to receive and manage alerts from Prometheus. + +The Alertmanager capabilities can be consumed by the dashboard in three different +ways: + +#. Use the notification receiver of the dashboard. + +#. Use the Prometheus Alertmanager API. + +#. Use both sources simultaneously. + +All three methods are going to notify you about alerts. You won't be notified +twice if you use both sources. + +#. Use the notification receiver of the dashboard: + + This allows you to get notifications as `configured + `_ from the Alertmanager. + You will get notified inside the dashboard once a notification is send out, + but you are not able to manage alerts. + + Add the dashboard receiver and the new route to your Alertmanager configuration. + This should look like:: + + route: + receiver: 'ceph-dashboard' + ... + receivers: + - name: 'ceph-dashboard' + webhook_configs: + - url: '/api/prometheus_receiver' + + + Please make sure that the Alertmanager considers your SSL certificate in terms + of the dashboard as valid. For more information about the correct + configuration checkout the ` documentation + `_. + +#. Use the API of the Prometheus Alertmanager + + This allows you to manage alerts. You will see all alerts, the Alertmanager + currently knows of, in the alerts listing. It can be found in the *Cluster* + submenu as *Alerts*. The alerts can be sorted by name, job, severity, + state and start time. Unfortunately it's not possible to know when an alert + was sent out through a notification by the Alertmanager based on your + configuration, that's why the dashboard will notify the user on any visible + change to an alert and will notify the changed alert. + + Currently it's not yet possible to silence an alert and expire an silenced + alert, but this is work in progress and will be added in a future release. + + To use it, specify the host and port of the Alertmanager server:: + + $ ceph dashboard set-alertmanager-api-host # default: '' + + For example:: + + $ ceph dashboard set-alertmanager-api-host 'http://localhost:9093' + + +#. Use both methods + + The different behaviors of both methods are configured in a way that they + should not disturb each other through annoying duplicated notifications + popping up. + Accessing the dashboard ^^^^^^^^^^^^^^^^^^^^^^^ @@ -470,6 +545,7 @@ scopes are: management. - **log**: include all features related to Ceph logs management. - **grafana**: include all features related to Grafana proxy. +- **prometheus**: include all features related to Prometheus alert management. - **dashboard-settings**: allows to change dashboard settings. A *role* specifies a set of mappings between a *security scope* and a set of diff --git a/src/pybind/mgr/dashboard/controllers/prometheus.py b/src/pybind/mgr/dashboard/controllers/prometheus.py new file mode 100644 index 00000000000..d233361a5b1 --- /dev/null +++ b/src/pybind/mgr/dashboard/controllers/prometheus.py @@ -0,0 +1,44 @@ +# -*- coding: utf-8 -*- +from __future__ import absolute_import + +from datetime import datetime +import json +import requests + +from . import Controller, ApiController, BaseController, RESTController, Endpoint +from ..security import Scope +from ..settings import Settings + + +@Controller('/api/prometheus_receiver', secure=False) +class PrometheusReceiver(BaseController): + # The receiver is needed in order to receive alert notifications (reports) + notifications = [] + + @Endpoint('POST', path='/') + def fetch_alert(self, **notification): + notification['notified'] = datetime.now().isoformat() + self.notifications.append(notification) + + +@ApiController('/prometheus', Scope.PROMETHEUS) +class Prometheus(RESTController): + def _get_api_url(self): + return Settings.ALERTMANAGER_API_HOST.rstrip('/') + '/api/v1' + + def _api_request(self, url_suffix, params=None): + url = self._get_api_url() + url_suffix + response = requests.request('GET', url, params=params) + payload = json.loads(response.content) + return payload['data'] if 'data' in payload else [] + + def list(self, **params): + return self._api_request('/alerts', params) + + @RESTController.Collection('POST') + def get_notifications_since(self, **last_notification): + notifications = PrometheusReceiver.notifications + if last_notification not in notifications: + return notifications[-1:] + index = notifications.index(last_notification) + return notifications[index + 1:] diff --git a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts index fe7ee88448f..fddc917b759 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/app-routing.module.ts @@ -13,6 +13,7 @@ import { HostsComponent } from './ceph/cluster/hosts/hosts.component'; import { LogsComponent } from './ceph/cluster/logs/logs.component'; import { MonitorComponent } from './ceph/cluster/monitor/monitor.component'; import { OsdListComponent } from './ceph/cluster/osd/osd-list/osd-list.component'; +import { PrometheusListComponent } from './ceph/cluster/prometheus/prometheus-list/prometheus-list.component'; import { DashboardComponent } from './ceph/dashboard/dashboard/dashboard.component'; import { PerformanceCounterComponent } from './ceph/performance-counter/performance-counter/performance-counter.component'; import { PoolFormComponent } from './ceph/pool/pool-form/pool-form.component'; @@ -109,6 +110,12 @@ const routes: Routes = [ canActivate: [AuthGuardService], data: { breadcrumbs: 'Cluster/Logs' } }, + { + path: 'alerts', + component: PrometheusListComponent, + canActivate: [AuthGuardService], + data: { breadcrumbs: 'Cluster/Alerts' } + }, { path: 'perf_counters/:type/:id', component: PerformanceCounterComponent, diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts index 2adb5a015af..a8aeb7d855f 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/cluster.module.ts @@ -27,6 +27,7 @@ import { OsdPerformanceHistogramComponent } from './osd/osd-performance-histogra import { OsdRecvSpeedModalComponent } from './osd/osd-recv-speed-modal/osd-recv-speed-modal.component'; import { OsdReweightModalComponent } from './osd/osd-reweight-modal/osd-reweight-modal.component'; import { OsdScrubModalComponent } from './osd/osd-scrub-modal/osd-scrub-modal.component'; +import { PrometheusListComponent } from './prometheus/prometheus-list/prometheus-list.component'; @NgModule({ entryComponents: [ @@ -65,6 +66,7 @@ import { OsdScrubModalComponent } from './osd/osd-scrub-modal/osd-scrub-modal.co OsdReweightModalComponent, CrushmapComponent, LogsComponent, + PrometheusListComponent, OsdRecvSpeedModalComponent ] }) diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.html b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.html new file mode 100644 index 00000000000..50ac7bda711 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.html @@ -0,0 +1,27 @@ + + + + + + + + + + + Source + + diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.scss b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.scss new file mode 100644 index 00000000000..e69de29bb2d diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.spec.ts new file mode 100644 index 00000000000..7901a05cb33 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.spec.ts @@ -0,0 +1,30 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { ComponentFixture, TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; +import { TabsModule } from 'ngx-bootstrap/tabs'; + +import { configureTestBed, i18nProviders } from '../../../../../testing/unit-test-helper'; +import { SharedModule } from '../../../../shared/shared.module'; +import { PrometheusListComponent } from './prometheus-list.component'; + +describe('PrometheusListComponent', () => { + let component: PrometheusListComponent; + let fixture: ComponentFixture; + + configureTestBed({ + imports: [HttpClientTestingModule, TabsModule.forRoot(), ToastModule.forRoot(), SharedModule], + declarations: [PrometheusListComponent], + providers: [i18nProviders] + }); + + beforeEach(() => { + fixture = TestBed.createComponent(PrometheusListComponent); + component = fixture.componentInstance; + fixture.detectChanges(); + }); + + it('should create', () => { + expect(component).toBeTruthy(); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.ts new file mode 100644 index 00000000000..722046e001c --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/ceph/cluster/prometheus/prometheus-list/prometheus-list.component.ts @@ -0,0 +1,70 @@ +import { Component, OnInit, TemplateRef, ViewChild } from '@angular/core'; +import { I18n } from '@ngx-translate/i18n-polyfill'; +import { CellTemplate } from '../../../../shared/enum/cell-template.enum'; +import { CdTableColumn } from '../../../../shared/models/cd-table-column'; +import { CdTableSelection } from '../../../../shared/models/cd-table-selection'; +import { CdDatePipe } from '../../../../shared/pipes/cd-date.pipe'; +import { PrometheusAlertService } from '../../../../shared/services/prometheus-alert.service'; + +@Component({ + selector: 'cd-prometheus-list', + templateUrl: './prometheus-list.component.html', + styleUrls: ['./prometheus-list.component.scss'] +}) +export class PrometheusListComponent implements OnInit { + @ViewChild('externalLinkTpl') + externalLinkTpl: TemplateRef; + columns: CdTableColumn[]; + selection = new CdTableSelection(); + customCss = { + 'label label-danger': 'active', + 'label label-warning': 'unprocessed', + 'label label-info': 'suppressed' + }; + + constructor( + // NotificationsComponent will refresh all alerts every 5s (No need to do it here as well) + public prometheusAlertService: PrometheusAlertService, + private i18n: I18n, + private cdDatePipe: CdDatePipe + ) {} + + ngOnInit() { + this.columns = [ + { + name: this.i18n('Name'), + prop: 'labels.alertname', + flexGrow: 2 + }, + { + name: this.i18n('Job'), + prop: 'labels.job', + flexGrow: 2 + }, + { + name: this.i18n('Severity'), + prop: 'labels.severity' + }, + { + name: this.i18n('State'), + prop: 'status.state', + cellTransformation: CellTemplate.classAdding + }, + { + name: this.i18n('Started'), + prop: 'startsAt', + pipe: this.cdDatePipe + }, + { + name: this.i18n('URL'), + prop: 'generatorURL', + sortable: false, + cellTemplate: this.externalLinkTpl + } + ]; + } + + updateSelection(selection: CdTableSelection) { + this.selection = selection; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html index 7b5713515e7..7083d73690a 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.html @@ -90,6 +90,12 @@ class="dropdown-item" routerLink="/logs">Logs +
  • + Alerts +
    • diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts index 87ce7e1c86e..4c304646a57 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/navigation/navigation.component.ts @@ -1,5 +1,6 @@ import { Component, OnInit } from '@angular/core'; +import { PrometheusService } from '../../../shared/api/prometheus.service'; import { Permissions } from '../../../shared/models/permissions'; import { AuthStorageService } from '../../../shared/services/auth-storage.service'; import { SummaryService } from '../../../shared/services/summary.service'; @@ -12,10 +13,13 @@ import { SummaryService } from '../../../shared/services/summary.service'; export class NavigationComponent implements OnInit { permissions: Permissions; summaryData: any; + isCollapsed = true; + prometheusConfigured = false; constructor( private authStorageService: AuthStorageService, + private prometheusService: PrometheusService, private summaryService: SummaryService ) { this.permissions = this.authStorageService.getPermissions(); @@ -28,6 +32,7 @@ export class NavigationComponent implements OnInit { } this.summaryData = data; }); + this.prometheusService.ifAlertmanagerConfigured(() => (this.prometheusConfigured = true)); } blockHealthColor() { diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts index 5b081d8d720..a11be2b8c68 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.spec.ts @@ -1,9 +1,14 @@ -import { ComponentFixture, TestBed } from '@angular/core/testing'; +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { ComponentFixture, fakeAsync, TestBed, tick } from '@angular/core/testing'; import { ToastModule } from 'ng2-toastr'; import { PopoverModule } from 'ngx-bootstrap/popover'; import { configureTestBed, i18nProviders } from '../../../../testing/unit-test-helper'; +import { PrometheusService } from '../../../shared/api/prometheus.service'; +import { AuthStorageService } from '../../../shared/services/auth-storage.service'; +import { PrometheusAlertService } from '../../../shared/services/prometheus-alert.service'; +import { PrometheusNotificationService } from '../../../shared/services/prometheus-notification.service'; import { SharedModule } from '../../../shared/shared.module'; import { NotificationsComponent } from './notifications.component'; @@ -12,7 +17,12 @@ describe('NotificationsComponent', () => { let fixture: ComponentFixture; configureTestBed({ - imports: [PopoverModule.forRoot(), SharedModule, ToastModule.forRoot()], + imports: [ + HttpClientTestingModule, + PopoverModule.forRoot(), + SharedModule, + ToastModule.forRoot() + ], declarations: [NotificationsComponent], providers: i18nProviders }); @@ -20,10 +30,60 @@ describe('NotificationsComponent', () => { beforeEach(() => { fixture = TestBed.createComponent(NotificationsComponent); component = fixture.componentInstance; - fixture.detectChanges(); }); it('should create', () => { + fixture.detectChanges(); expect(component).toBeTruthy(); }); + + describe('prometheus alert handling', () => { + let prometheusAlertService: PrometheusAlertService; + let prometheusNotificationService: PrometheusNotificationService; + let prometheusAccessAllowed: boolean; + + const expectPrometheusServicesToBeCalledTimes = (n: number) => { + expect(prometheusNotificationService.refresh).toHaveBeenCalledTimes(n); + expect(prometheusAlertService.refresh).toHaveBeenCalledTimes(n); + }; + + beforeEach(() => { + prometheusAccessAllowed = true; + spyOn(TestBed.get(AuthStorageService), 'getPermissions').and.callFake(() => ({ + prometheus: { read: prometheusAccessAllowed } + })); + + spyOn(TestBed.get(PrometheusService), 'ifAlertmanagerConfigured').and.callFake((fn) => fn()); + + prometheusAlertService = TestBed.get(PrometheusAlertService); + spyOn(prometheusAlertService, 'refresh').and.stub(); + + prometheusNotificationService = TestBed.get(PrometheusNotificationService); + spyOn(prometheusNotificationService, 'refresh').and.stub(); + }); + + it('should not refresh prometheus services if not allowed', () => { + prometheusAccessAllowed = false; + fixture.detectChanges(); + + expectPrometheusServicesToBeCalledTimes(0); + }); + it('should first refresh prometheus notifications and alerts during init', () => { + fixture.detectChanges(); + + expect(prometheusAlertService.refresh).toHaveBeenCalledTimes(1); + expectPrometheusServicesToBeCalledTimes(1); + }); + + it('should refresh prometheus services every 5s', fakeAsync(() => { + fixture.detectChanges(); + + expectPrometheusServicesToBeCalledTimes(1); + tick(5000); + expectPrometheusServicesToBeCalledTimes(2); + tick(15000); + expectPrometheusServicesToBeCalledTimes(5); + component.ngOnDestroy(); + })); + }); }); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts index d96b208de51..ad433c485a9 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/core/navigation/notifications/notifications.component.ts @@ -1,30 +1,58 @@ -import { Component, OnInit } from '@angular/core'; +import { Component, NgZone, OnDestroy, OnInit } from '@angular/core'; import * as _ from 'lodash'; import { NotificationType } from '../../../shared/enum/notification-type.enum'; import { CdNotification } from '../../../shared/models/cd-notification'; +import { AuthStorageService } from '../../../shared/services/auth-storage.service'; import { NotificationService } from '../../../shared/services/notification.service'; +import { PrometheusAlertService } from '../../../shared/services/prometheus-alert.service'; +import { PrometheusNotificationService } from '../../../shared/services/prometheus-notification.service'; @Component({ selector: 'cd-notifications', templateUrl: './notifications.component.html', styleUrls: ['./notifications.component.scss'] }) -export class NotificationsComponent implements OnInit { +export class NotificationsComponent implements OnInit, OnDestroy { notifications: CdNotification[]; - notificationType = NotificationType; + private interval: number; - constructor(private notificationService: NotificationService) { + constructor( + private notificationService: NotificationService, + private prometheusNotificationService: PrometheusNotificationService, + private authStorageService: AuthStorageService, + private prometheusAlertService: PrometheusAlertService, + private ngZone: NgZone + ) { this.notifications = []; } + ngOnDestroy() { + window.clearInterval(this.interval); + } + ngOnInit() { + if (this.authStorageService.getPermissions().prometheus.read) { + this.triggerPrometheusAlerts(); + this.ngZone.runOutsideAngular(() => { + this.interval = window.setInterval(() => { + this.ngZone.run(() => { + this.triggerPrometheusAlerts(); + }); + }, 5000); + }); + } this.notificationService.data$.subscribe((notifications: CdNotification[]) => { this.notifications = _.orderBy(notifications, ['timestamp'], ['desc']); }); } + private triggerPrometheusAlerts() { + this.prometheusAlertService.refresh(); + this.prometheusNotificationService.refresh(); + } + removeAll() { this.notificationService.removeAll(); } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.spec.ts new file mode 100644 index 00000000000..0a0c49fc3be --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.spec.ts @@ -0,0 +1,70 @@ +import { HttpClientTestingModule, HttpTestingController } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { configureTestBed } from '../../../testing/unit-test-helper'; +import { PrometheusService } from './prometheus.service'; +import { SettingsService } from './settings.service'; + +describe('PrometheusService', () => { + let service: PrometheusService; + let httpTesting: HttpTestingController; + + configureTestBed({ + providers: [PrometheusService, SettingsService], + imports: [HttpClientTestingModule] + }); + + beforeEach(() => { + service = TestBed.get(PrometheusService); + httpTesting = TestBed.get(HttpTestingController); + }); + + afterEach(() => { + httpTesting.verify(); + }); + + it('should be created', () => { + expect(service).toBeTruthy(); + }); + + it('should call list', () => { + service.list().subscribe(); + const req = httpTesting.expectOne('api/prometheus'); + expect(req.request.method).toBe('GET'); + }); + + it('should call getNotificationSince', () => { + service.getNotificationSince({}).subscribe(); + const req = httpTesting.expectOne('api/prometheus/get_notifications_since'); + expect(req.request.method).toBe('POST'); + }); + + describe('ifAlertmanagerConfigured', () => { + let x: any; + + const receiveConfig = (value) => { + const req = httpTesting.expectOne('api/settings/alertmanager-api-host'); + expect(req.request.method).toBe('GET'); + req.flush({ value }); + }; + + beforeEach(() => { + x = false; + TestBed.get(SettingsService)['settings'] = {}; + }); + + it('changes x in a valid case', () => { + service.ifAlertmanagerConfigured((v) => (x = v)); + expect(x).toBe(false); + const host = 'http://localhost:9093'; + receiveConfig(host); + expect(x).toBe(host); + }); + + it('does not change x in a invalid case', () => { + service.ifAlertmanagerConfigured((v) => (x = v)); + receiveConfig(''); + expect(x).toBe(false); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts new file mode 100644 index 00000000000..0fd288e9c01 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/api/prometheus.service.ts @@ -0,0 +1,32 @@ +import { HttpClient } from '@angular/common/http'; +import { Injectable } from '@angular/core'; + +import { Observable } from 'rxjs'; + +import { PrometheusAlert, PrometheusNotification } from '../models/prometheus-alerts'; +import { ApiModule } from './api.module'; +import { SettingsService } from './settings.service'; + +@Injectable({ + providedIn: ApiModule +}) +export class PrometheusService { + private baseURL = 'api/prometheus'; + + constructor(private http: HttpClient, private settingsService: SettingsService) {} + + ifAlertmanagerConfigured(fn): void { + this.settingsService.ifSettingConfigured('api/settings/alertmanager-api-host', fn); + } + + list(params = {}): Observable { + return this.http.get(this.baseURL, { params }); + } + + getNotificationSince(notification): Observable { + return this.http.post( + `${this.baseURL}/get_notifications_since`, + notification + ); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permission.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permission.spec.ts new file mode 100644 index 00000000000..e2725bc856d --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permission.spec.ts @@ -0,0 +1,60 @@ +import { Permissions } from './permissions'; + +describe('cd-notification classes', () => { + it('should show empty permissions', () => { + expect(new Permissions({})).toEqual({ + cephfs: { create: false, delete: false, read: false, update: false }, + configOpt: { create: false, delete: false, read: false, update: false }, + grafana: { create: false, delete: false, read: false, update: false }, + hosts: { create: false, delete: false, read: false, update: false }, + iscsi: { create: false, delete: false, read: false, update: false }, + log: { create: false, delete: false, read: false, update: false }, + manager: { create: false, delete: false, read: false, update: false }, + monitor: { create: false, delete: false, read: false, update: false }, + osd: { create: false, delete: false, read: false, update: false }, + pool: { create: false, delete: false, read: false, update: false }, + prometheus: { create: false, delete: false, read: false, update: false }, + rbdImage: { create: false, delete: false, read: false, update: false }, + rbdMirroring: { create: false, delete: false, read: false, update: false }, + rgw: { create: false, delete: false, read: false, update: false }, + user: { create: false, delete: false, read: false, update: false } + }); + }); + + it('should show full permissions', () => { + const fullyGranted = { + cephfs: ['create', 'read', 'update', 'delete'], + 'config-opt': ['create', 'read', 'update', 'delete'], + grafana: ['create', 'read', 'update', 'delete'], + hosts: ['create', 'read', 'update', 'delete'], + iscsi: ['create', 'read', 'update', 'delete'], + log: ['create', 'read', 'update', 'delete'], + manager: ['create', 'read', 'update', 'delete'], + monitor: ['create', 'read', 'update', 'delete'], + osd: ['create', 'read', 'update', 'delete'], + pool: ['create', 'read', 'update', 'delete'], + prometheus: ['create', 'read', 'update', 'delete'], + 'rbd-image': ['create', 'read', 'update', 'delete'], + 'rbd-mirroring': ['create', 'read', 'update', 'delete'], + rgw: ['create', 'read', 'update', 'delete'], + user: ['create', 'read', 'update', 'delete'] + }; + expect(new Permissions(fullyGranted)).toEqual({ + cephfs: { create: true, delete: true, read: true, update: true }, + configOpt: { create: true, delete: true, read: true, update: true }, + grafana: { create: true, delete: true, read: true, update: true }, + hosts: { create: true, delete: true, read: true, update: true }, + iscsi: { create: true, delete: true, read: true, update: true }, + log: { create: true, delete: true, read: true, update: true }, + manager: { create: true, delete: true, read: true, update: true }, + monitor: { create: true, delete: true, read: true, update: true }, + osd: { create: true, delete: true, read: true, update: true }, + pool: { create: true, delete: true, read: true, update: true }, + prometheus: { create: true, delete: true, read: true, update: true }, + rbdImage: { create: true, delete: true, read: true, update: true }, + rbdMirroring: { create: true, delete: true, read: true, update: true }, + rgw: { create: true, delete: true, read: true, update: true }, + user: { create: true, delete: true, read: true, update: true } + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts index 0938e264b65..0d8d580266c 100644 --- a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/permissions.ts @@ -5,10 +5,9 @@ export class Permission { delete: boolean; constructor(serverPermission: Array = []) { - this.read = serverPermission.indexOf('read') !== -1; - this.create = serverPermission.indexOf('create') !== -1; - this.update = serverPermission.indexOf('update') !== -1; - this.delete = serverPermission.indexOf('delete') !== -1; + ['read', 'create', 'update', 'delete'].forEach( + (permission) => (this[permission] = serverPermission.includes(permission)) + ); } } @@ -27,6 +26,7 @@ export class Permissions { log: Permission; user: Permission; grafana: Permission; + prometheus: Permission; constructor(serverPermissions: any) { this.hosts = new Permission(serverPermissions['hosts']); @@ -43,5 +43,6 @@ export class Permissions { this.log = new Permission(serverPermissions['log']); this.user = new Permission(serverPermissions['user']); this.grafana = new Permission(serverPermissions['grafana']); + this.prometheus = new Permission(serverPermissions['prometheus']); } } diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/models/prometheus-alerts.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/prometheus-alerts.ts new file mode 100644 index 00000000000..204333110d5 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/models/prometheus-alerts.ts @@ -0,0 +1,52 @@ +class CommonAlert { + labels: { + alertname: string; + instance: string; + job: string; + severity: string; + }; + annotations: { + description: string; + summary: string; + }; + startsAt: string; + endsAt: string; + generatorURL: string; +} + +export class PrometheusAlert extends CommonAlert { + status: { + state: 'unprocessed' | 'active' | 'suppressed'; + silencedBy: null | string[]; + inhibitedBy: null | string[]; + }; + receivers: string[]; + fingerprint: string; +} + +export class PrometheusNotificationAlert extends CommonAlert { + status: 'firing' | 'resolved'; +} + +export class PrometheusNotification { + status: 'firing' | 'resolved'; + groupLabels: object; + commonAnnotations: object; + groupKey: string; + notified: string; + alerts: PrometheusNotificationAlert[]; + version: string; + receiver: string; + externalURL: string; + commonLabels: { + severity: string; + }; +} + +export class PrometheusCustomAlert { + status: 'resolved' | 'unprocessed' | 'active' | 'suppressed'; + name: string; + url: string; + summary: string; + fingerprint?: string | boolean; +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.spec.ts new file mode 100644 index 00000000000..a0a5b49403f --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.spec.ts @@ -0,0 +1,98 @@ +import { TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; + +import { + configureTestBed, + i18nProviders, + PrometheusHelper +} from '../../../testing/unit-test-helper'; +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusCustomAlert } from '../models/prometheus-alerts'; +import { SharedModule } from '../shared.module'; +import { NotificationService } from './notification.service'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; + +describe('PrometheusAlertFormatter', () => { + let service: PrometheusAlertFormatter; + let notificationService: NotificationService; + let prometheus: PrometheusHelper; + + configureTestBed({ + imports: [ToastModule.forRoot(), SharedModule], + providers: [PrometheusAlertFormatter, i18nProviders] + }); + + beforeEach(() => { + prometheus = new PrometheusHelper(); + service = TestBed.get(PrometheusAlertFormatter); + notificationService = TestBed.get(NotificationService); + spyOn(notificationService, 'queueNotifications').and.stub(); + }); + + it('should create', () => { + expect(service).toBeTruthy(); + }); + + describe('sendNotifications', () => { + it('should not call queue notifications with no notification', () => { + service.sendNotifications([]); + expect(notificationService.queueNotifications).not.toHaveBeenCalled(); + }); + + it('should call queue notifications with notifications', () => { + const notifications = [new CdNotificationConfig(NotificationType.success, 'test')]; + service.sendNotifications(notifications); + expect(notificationService.queueNotifications).toHaveBeenCalledWith(notifications); + }); + }); + + describe('convertToCustomAlert', () => { + it('converts PrometheusAlert', () => { + expect(service.convertToCustomAlerts([prometheus.createAlert('Something')])).toEqual([ + { + status: 'active', + name: 'Something', + summary: 'Something is active', + url: 'http://Something', + fingerprint: 'Something' + } as PrometheusCustomAlert + ]); + }); + + it('converts PrometheusNotificationAlert', () => { + expect( + service.convertToCustomAlerts([prometheus.createNotificationAlert('Something')]) + ).toEqual([ + { + fingerprint: false, + status: 'active', + name: 'Something', + summary: 'Something is firing', + url: 'http://Something' + } as PrometheusCustomAlert + ]); + }); + }); + + it('converts custom alert into notification', () => { + const alert: PrometheusCustomAlert = { + status: 'active', + name: 'Some alert', + summary: 'Some alert is active', + url: 'http://some-alert', + fingerprint: '42' + }; + expect(service.convertAlertToNotification(alert)).toEqual( + new CdNotificationConfig( + NotificationType.error, + 'Some alert (active)', + 'Some alert is active ' + + '', + undefined, + 'Prometheus' + ) + ); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.ts new file mode 100644 index 00000000000..8fdc5ddb68f --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert-formatter.ts @@ -0,0 +1,76 @@ +import { Injectable } from '@angular/core'; + +import * as _ from 'lodash'; + +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { + PrometheusAlert, + PrometheusCustomAlert, + PrometheusNotificationAlert +} from '../models/prometheus-alerts'; +import { NotificationService } from './notification.service'; +import { ServicesModule } from './services.module'; + +@Injectable({ + providedIn: ServicesModule +}) +export class PrometheusAlertFormatter { + constructor(private notificationService: NotificationService) {} + + sendNotifications(notifications: CdNotificationConfig[]) { + if (notifications.length > 0) { + this.notificationService.queueNotifications(notifications); + } + } + + convertToCustomAlerts( + alerts: (PrometheusNotificationAlert | PrometheusAlert)[] + ): PrometheusCustomAlert[] { + return _.uniqWith( + alerts.map((alert) => { + return { + status: _.isObject(alert.status) + ? (alert as PrometheusAlert).status.state + : this.getPrometheusNotificationStatus(alert as PrometheusNotificationAlert), + name: alert.labels.alertname, + url: alert.generatorURL, + summary: alert.annotations.summary, + fingerprint: _.isObject(alert.status) && (alert as PrometheusAlert).fingerprint + }; + }), + _.isEqual + ) as PrometheusCustomAlert[]; + } + + /* + * This is needed because NotificationAlerts don't use 'active' + */ + private getPrometheusNotificationStatus(alert: PrometheusNotificationAlert): string { + const state = alert.status; + return state === 'firing' ? 'active' : state; + } + + convertAlertToNotification(alert: PrometheusCustomAlert): CdNotificationConfig { + return new CdNotificationConfig( + this.formatType(alert.status), + `${alert.name} (${alert.status})`, + this.appendSourceLink(alert, alert.summary), + undefined, + 'Prometheus' + ); + } + + private formatType(status: string): NotificationType { + const types = { + error: ['firing', 'active'], + info: ['suppressed', 'unprocessed'], + success: ['resolved'] + }; + return NotificationType[_.findKey(types, (type) => type.includes(status))]; + } + + private appendSourceLink(alert: PrometheusCustomAlert, message: string): string { + return `${message} `; + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.spec.ts new file mode 100644 index 00000000000..36fe4c66fbd --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.spec.ts @@ -0,0 +1,154 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; +import { of } from 'rxjs'; + +import { + configureTestBed, + i18nProviders, + PrometheusHelper +} from '../../../testing/unit-test-helper'; +import { PrometheusService } from '../api/prometheus.service'; +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusAlert } from '../models/prometheus-alerts'; +import { SharedModule } from '../shared.module'; +import { NotificationService } from './notification.service'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { PrometheusAlertService } from './prometheus-alert.service'; + +describe('PrometheusAlertService', () => { + let service: PrometheusAlertService; + let notificationService: NotificationService; + let alerts: PrometheusAlert[]; + let prometheusService: PrometheusService; + let prometheus: PrometheusHelper; + + configureTestBed({ + imports: [ToastModule.forRoot(), SharedModule, HttpClientTestingModule], + providers: [PrometheusAlertService, PrometheusAlertFormatter, i18nProviders] + }); + + beforeEach(() => { + prometheus = new PrometheusHelper(); + }); + + it('should create', () => { + expect(TestBed.get(PrometheusAlertService)).toBeTruthy(); + }); + + it('tests error case ', () => { + const resp = { status: 500, error: {} }; + service = new PrometheusAlertService(null, { + ifAlertmanagerConfigured: (fn) => fn(), + list: () => ({ subscribe: (fn, err) => err(resp) }) + }); + + expect(service['connected']).toBe(true); + service.refresh(); + expect(service['connected']).toBe(false); + expect(resp['application']).toBe('Prometheus'); + expect(resp.error['detail']).toBe( + 'Please check if Prometheus Alertmanager is still running' + ); + }); + + describe('refresh', () => { + beforeEach(() => { + service = TestBed.get(PrometheusAlertService); + service['alerts'] = []; + service['canAlertsBeNotified'] = false; + + spyOn(window, 'setTimeout').and.callFake((fn: Function) => fn()); + + notificationService = TestBed.get(NotificationService); + spyOn(notificationService, 'queueNotifications').and.callThrough(); + spyOn(notificationService, 'show').and.stub(); + + prometheusService = TestBed.get(PrometheusService); + spyOn(prometheusService, 'ifAlertmanagerConfigured').and.callFake((fn) => fn()); + spyOn(prometheusService, 'list').and.callFake(() => of(alerts)); + + alerts = [prometheus.createAlert('alert0')]; + service.refresh(); + }); + + it('should not notify on first call', () => { + expect(notificationService.show).not.toHaveBeenCalled(); + }); + + it('should not notify with no change', () => { + service.refresh(); + expect(notificationService.show).not.toHaveBeenCalled(); + }); + + it('should notify on alert change', () => { + alerts = [prometheus.createAlert('alert0', 'suppressed')]; + service.refresh(); + expect(notificationService.queueNotifications).toHaveBeenCalledWith([ + new CdNotificationConfig( + NotificationType.info, + 'alert0 (suppressed)', + 'alert0 is suppressed ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('should notify on a new alert', () => { + alerts = [prometheus.createAlert('alert1'), prometheus.createAlert('alert0')]; + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(1); + expect(notificationService.show).toHaveBeenCalledWith( + new CdNotificationConfig( + NotificationType.error, + 'alert1 (active)', + 'alert1 is active ' + prometheus.createLink('http://alert1'), + undefined, + 'Prometheus' + ) + ); + }); + + it('should notify a resolved alert if it is not there anymore', () => { + alerts = []; + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(1); + expect(notificationService.show).toHaveBeenCalledWith( + new CdNotificationConfig( + NotificationType.success, + 'alert0 (resolved)', + 'alert0 is active ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ); + }); + + it('should call multiple times for multiple changes', () => { + const alert1 = prometheus.createAlert('alert1'); + alerts.push(alert1); + service.refresh(); + alerts = [alert1, prometheus.createAlert('alert2')]; + service.refresh(); + expect(notificationService.queueNotifications).toHaveBeenCalledWith([ + new CdNotificationConfig( + NotificationType.error, + 'alert2 (active)', + 'alert2 is active ' + prometheus.createLink('http://alert2'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.success, + 'alert0 (resolved)', + 'alert0 is active ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts new file mode 100644 index 00000000000..9a6d26c88e2 --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-alert.service.ts @@ -0,0 +1,73 @@ +import { Injectable } from '@angular/core'; + +import * as _ from 'lodash'; + +import { PrometheusService } from '../api/prometheus.service'; +import { PrometheusAlert, PrometheusCustomAlert } from '../models/prometheus-alerts'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { ServicesModule } from './services.module'; + +@Injectable({ + providedIn: ServicesModule +}) +export class PrometheusAlertService { + private canAlertsBeNotified = false; + private connected = true; + alerts: PrometheusAlert[] = []; + + constructor( + private alertFormatter: PrometheusAlertFormatter, + private prometheusService: PrometheusService + ) {} + + refresh() { + this.prometheusService.ifAlertmanagerConfigured((url) => { + if (this.connected) { + this.prometheusService.list().subscribe( + (alerts) => this.handleAlerts(alerts), + (resp) => { + const errorMsg = `Please check if Prometheus Alertmanager is still running`; + resp['application'] = 'Prometheus'; + if (resp.status === 500) { + this.connected = false; + resp.error.detail = errorMsg; + } + } + ); + } + }); + } + + private handleAlerts(alerts: PrometheusAlert[]) { + if (this.canAlertsBeNotified) { + this.notifyOnAlertChanges(alerts, this.alerts); + } + this.alerts = alerts; + this.canAlertsBeNotified = true; + } + + private notifyOnAlertChanges(alerts: PrometheusAlert[], oldAlerts: PrometheusAlert[]) { + const changedAlerts = this.getChangedAlerts( + this.alertFormatter.convertToCustomAlerts(alerts), + this.alertFormatter.convertToCustomAlerts(oldAlerts) + ); + const notifications = changedAlerts.map((alert) => + this.alertFormatter.convertAlertToNotification(alert) + ); + this.alertFormatter.sendNotifications(notifications); + } + + private getChangedAlerts(alerts: PrometheusCustomAlert[], oldAlerts: PrometheusCustomAlert[]) { + const updatedAndNew = _.differenceWith(alerts, oldAlerts, _.isEqual); + return updatedAndNew.concat(this.getVanishedAlerts(alerts, oldAlerts)); + } + + private getVanishedAlerts(alerts: PrometheusCustomAlert[], oldAlerts: PrometheusCustomAlert[]) { + return _.differenceWith(oldAlerts, alerts, (a, b) => a.fingerprint === b.fingerprint).map( + (alert) => { + alert.status = 'resolved'; + return alert; + } + ); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.spec.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.spec.ts new file mode 100644 index 00000000000..d395e84064b --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.spec.ts @@ -0,0 +1,197 @@ +import { HttpClientTestingModule } from '@angular/common/http/testing'; +import { TestBed } from '@angular/core/testing'; + +import { ToastModule } from 'ng2-toastr'; +import { of } from 'rxjs'; + +import { + configureTestBed, + i18nProviders, + PrometheusHelper +} from '../../../testing/unit-test-helper'; +import { PrometheusService } from '../api/prometheus.service'; +import { NotificationType } from '../enum/notification-type.enum'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusNotification } from '../models/prometheus-alerts'; +import { SharedModule } from '../shared.module'; +import { NotificationService } from './notification.service'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { PrometheusNotificationService } from './prometheus-notification.service'; + +describe('PrometheusNotificationService', () => { + let service: PrometheusNotificationService; + let notificationService: NotificationService; + let notifications: PrometheusNotification[]; + let prometheusService: PrometheusService; + let prometheus: PrometheusHelper; + let shown: CdNotificationConfig[]; + + configureTestBed({ + imports: [ToastModule.forRoot(), SharedModule, HttpClientTestingModule], + providers: [PrometheusNotificationService, PrometheusAlertFormatter, i18nProviders] + }); + + beforeEach(() => { + prometheus = new PrometheusHelper(); + + service = TestBed.get(PrometheusNotificationService); + service['notifications'] = []; + + notificationService = TestBed.get(NotificationService); + spyOn(notificationService, 'queueNotifications').and.callThrough(); + shown = []; + spyOn(notificationService, 'show').and.callFake((n) => shown.push(n)); + + spyOn(window, 'setTimeout').and.callFake((fn: Function) => fn()); + + prometheusService = TestBed.get(PrometheusService); + spyOn(prometheusService, 'getNotificationSince').and.callFake(() => of(notifications)); + + notifications = [prometheus.createNotification()]; + }); + + it('should create', () => { + expect(service).toBeTruthy(); + }); + + describe('getLastNotification', () => { + it('returns an empty object on the first call', () => { + service.refresh(); + expect(prometheusService.getNotificationSince).toHaveBeenCalledWith({}); + expect(service['notifications'].length).toBe(1); + }); + + it('returns last notification on any other call', () => { + service.refresh(); + notifications = [prometheus.createNotification(1, 'resolved')]; + service.refresh(); + expect(prometheusService.getNotificationSince).toHaveBeenCalledWith( + service['notifications'][0] + ); + expect(service['notifications'].length).toBe(2); + + notifications = [prometheus.createNotification(2)]; + service.refresh(); + notifications = [prometheus.createNotification(3, 'resolved')]; + service.refresh(); + expect(prometheusService.getNotificationSince).toHaveBeenCalledWith( + service['notifications'][2] + ); + expect(service['notifications'].length).toBe(4); + }); + }); + + it('notifies not on the first call', () => { + service.refresh(); + expect(notificationService.show).not.toHaveBeenCalled(); + }); + + describe('looks of fired notifications', () => { + beforeEach(() => { + service.refresh(); + service.refresh(); + shown = []; + }); + + it('notifies on the second call', () => { + expect(notificationService.show).toHaveBeenCalledTimes(1); + }); + + it('notify looks on single notification with single alert like', () => { + expect(notificationService.queueNotifications).toHaveBeenCalledWith([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('raises multiple pop overs for a single notification with multiple alerts', () => { + notifications[0].alerts.push(prometheus.createNotificationAlert('alert1', 'resolved')); + service.refresh(); + expect(shown).toEqual([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.success, + 'alert1 (resolved)', + 'alert1 is resolved ' + prometheus.createLink('http://alert1'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('should raise multiple notifications if they do not look like each other', () => { + notifications[0].alerts.push(prometheus.createNotificationAlert('alert1')); + notifications.push(prometheus.createNotification()); + notifications[1].alerts.push(prometheus.createNotificationAlert('alert2')); + service.refresh(); + expect(shown).toEqual([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.error, + 'alert1 (active)', + 'alert1 is firing ' + prometheus.createLink('http://alert1'), + undefined, + 'Prometheus' + ), + new CdNotificationConfig( + NotificationType.error, + 'alert2 (active)', + 'alert2 is firing ' + prometheus.createLink('http://alert2'), + undefined, + 'Prometheus' + ) + ]); + }); + + it('only shows toasties if it got new data', () => { + expect(notificationService.show).toHaveBeenCalledTimes(1); + notifications = []; + service.refresh(); + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(1); + notifications = [prometheus.createNotification()]; + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(2); + service.refresh(); + expect(notificationService.show).toHaveBeenCalledTimes(3); + }); + + it('filters out duplicated and non user visible changes in notifications', () => { + // Return 2 notifications with 3 duplicated alerts and 1 non visible changed alert + const secondAlert = prometheus.createNotificationAlert('alert0'); + secondAlert.endsAt = new Date().toString(); // Should be ignored as it's not visible + notifications[0].alerts.push(secondAlert); + notifications.push(prometheus.createNotification()); + notifications[1].alerts.push(prometheus.createNotificationAlert('alert0')); + notifications[1].notified = 'by somebody else'; + service.refresh(); + + expect(shown).toEqual([ + new CdNotificationConfig( + NotificationType.error, + 'alert0 (active)', + 'alert0 is firing ' + prometheus.createLink('http://alert0'), + undefined, + 'Prometheus' + ) + ]); + }); + }); +}); diff --git a/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.ts b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.ts new file mode 100644 index 00000000000..b931c26e09b --- /dev/null +++ b/src/pybind/mgr/dashboard/frontend/src/app/shared/services/prometheus-notification.service.ts @@ -0,0 +1,52 @@ +import { Injectable } from '@angular/core'; + +import * as _ from 'lodash'; + +import { PrometheusService } from '../api/prometheus.service'; +import { CdNotificationConfig } from '../models/cd-notification'; +import { PrometheusNotification } from '../models/prometheus-alerts'; +import { PrometheusAlertFormatter } from './prometheus-alert-formatter'; +import { ServicesModule } from './services.module'; + +@Injectable({ + providedIn: ServicesModule +}) +export class PrometheusNotificationService { + private notifications: PrometheusNotification[]; + + constructor( + private alertFormatter: PrometheusAlertFormatter, + private prometheusService: PrometheusService + ) { + this.notifications = []; + } + + refresh() { + const last = this.getLastNotification(); + this.prometheusService + .getNotificationSince(last) + .subscribe((notifications) => this.handleNotifications(notifications)); + } + + private getLastNotification() { + return _.last(this.notifications) || {}; + } + + private handleNotifications(notifications: PrometheusNotification[]) { + if (notifications.length === 0) { + return; + } + if (this.notifications.length > 0) { + this.alertFormatter.sendNotifications( + _.flatten(notifications.map((notification) => this.formatNotification(notification))) + ); + } + this.notifications = this.notifications.concat(notifications); + } + + private formatNotification(notification: PrometheusNotification): CdNotificationConfig[] { + return this.alertFormatter + .convertToCustomAlerts(notification.alerts) + .map((alert) => this.alertFormatter.convertAlertToNotification(alert)); + } +} diff --git a/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts b/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts index b294aab4e65..e4dcd176deb 100644 --- a/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts +++ b/src/pybind/mgr/dashboard/frontend/src/testing/unit-test-helper.ts @@ -9,6 +9,11 @@ import * as _ from 'lodash'; import { TableActionsComponent } from '../app/shared/datatable/table-actions/table-actions.component'; import { CdFormGroup } from '../app/shared/forms/cd-form-group'; import { Permission } from '../app/shared/models/permissions'; +import { + PrometheusAlert, + PrometheusNotification, + PrometheusNotificationAlert +} from '../app/shared/models/prometheus-alerts'; import { _DEV_ } from '../unit-test-configuration'; export function configureTestBed(configuration, useOldMethod?) { @@ -182,6 +187,48 @@ export class FormHelper { } } +export class PrometheusHelper { + createAlert(name, state = 'active', timeMultiplier = 1) { + return { + fingerprint: name, + status: { state }, + labels: { + alertname: name + }, + annotations: { + summary: `${name} is ${state}` + }, + generatorURL: `http://${name}`, + startsAt: new Date(new Date('2022-02-22').getTime() * timeMultiplier).toString() + } as PrometheusAlert; + } + + createNotificationAlert(name, status = 'firing') { + return { + status: status, + labels: { + alertname: name + }, + annotations: { + summary: `${name} is ${status}` + }, + generatorURL: `http://${name}` + } as PrometheusNotificationAlert; + } + + createNotification(alertNumber = 1, status = 'firing') { + const alerts = []; + for (let i = 0; i < alertNumber; i++) { + alerts.push(this.createNotificationAlert('alert' + i, status)); + } + return { alerts, status } as PrometheusNotification; + } + + createLink(url) { + return ``; + } +} + const XLIFF = ` diff --git a/src/pybind/mgr/dashboard/security.py b/src/pybind/mgr/dashboard/security.py index 64209e7f445..501c07f1789 100644 --- a/src/pybind/mgr/dashboard/security.py +++ b/src/pybind/mgr/dashboard/security.py @@ -23,6 +23,7 @@ class Scope(object): MANAGER = "manager" LOG = "log" GRAFANA = "grafana" + PROMETHEUS = "prometheus" USER = "user" DASHBOARD_SETTINGS = "dashboard-settings" diff --git a/src/pybind/mgr/dashboard/settings.py b/src/pybind/mgr/dashboard/settings.py index 73c490297e7..0bae36104af 100644 --- a/src/pybind/mgr/dashboard/settings.py +++ b/src/pybind/mgr/dashboard/settings.py @@ -43,6 +43,10 @@ class Options(object): # Orchestrator settings ORCHESTRATOR_BACKEND = ('', str) + # Prometheus settings + PROMETHEUS_API_HOST = ('', str) # Not in use ATM + ALERTMANAGER_API_HOST = ('', str) + @staticmethod def has_default_value(name): return getattr(Settings, name, None) is None or \ diff --git a/src/pybind/mgr/dashboard/tests/test_prometheus.py b/src/pybind/mgr/dashboard/tests/test_prometheus.py new file mode 100644 index 00000000000..5961187c8ce --- /dev/null +++ b/src/pybind/mgr/dashboard/tests/test_prometheus.py @@ -0,0 +1,67 @@ +# -*- coding: utf-8 -*- +from .. import mgr +from ..controllers import BaseController, Controller +from ..controllers.prometheus import Prometheus, PrometheusReceiver + +from .helper import ControllerTestCase + + +@Controller('alertmanager/mocked/api/v1/alerts', secure=False) +class AlertManagerMockInstance(BaseController): + def __call__(self, path, **params): + return 'Some Api {}'.format(path) + + +class PrometheusControllerTest(ControllerTestCase): + @classmethod + def setup_server(cls): + settings = { + 'ALERTMANAGER_API_HOST': 'http://localhost:{}/alertmanager/mocked/'.format(54583) + } + mgr.get_module_option.side_effect = settings.get + Prometheus._cp_config['tools.authenticate.on'] = False # pylint: disable=protected-access + cls.setup_controllers([AlertManagerMockInstance, Prometheus, PrometheusReceiver]) + + def test_list(self): + self._get('/api/prometheus') + self.assertStatus(200) + + def test_post_on_receiver(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + self.assertEqual(len(PrometheusReceiver.notifications), 1) + notification = PrometheusReceiver.notifications[0] + self.assertEqual(notification['name'], 'foo') + self.assertTrue(len(notification['notified']) > 20) + + def test_get_last_notification_with_empty_notifications(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + self._post('/api/prometheus_receiver', {'name': 'bar'}) + self._post('/api/prometheus/get_notifications_since', {}) + self.assertStatus(200) + last = PrometheusReceiver.notifications[1] + self.assertEqual(self.jsonBody(), [last]) + + def test_get_no_notification_since_with_last_notification(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + notification = PrometheusReceiver.notifications[0] + self._post('/api/prometheus/get_notifications_since', notification) + self.assertBody('[]') + + def test_get_empty_list_with_no_notifications(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus/get_notifications_since', {}) + self.assertEqual(self.jsonBody(), []) + + def test_get_notifications_since_last_notification(self): + PrometheusReceiver.notifications = [] + self._post('/api/prometheus_receiver', {'name': 'foobar'}) + next_to_last = PrometheusReceiver.notifications[0] + self._post('/api/prometheus_receiver', {'name': 'foo'}) + self._post('/api/prometheus_receiver', {'name': 'bar'}) + self._post('/api/prometheus/get_notifications_since', next_to_last) + foreLast = PrometheusReceiver.notifications[1] + last = PrometheusReceiver.notifications[2] + self.assertEqual(self.jsonBody(), [foreLast, last])