From 4efb502aab22d912fb70ee1588a2b69df2fc3e32 Mon Sep 17 00:00:00 2001 From: Andrey Burtasov <76874959+aburtasov@users.noreply.github.com> Date: Mon, 17 Jun 2024 12:05:52 +0300 Subject: [PATCH] Add label user to process collector (#1472) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Jan-Otto Kröpke Co-authored-by: Jan-Otto Kröpke --- docs/collector.process.md | 61 ++++++---- pkg/collector/process/process.go | 198 +++++++++++++++++-------------- 2 files changed, 147 insertions(+), 112 deletions(-) diff --git a/docs/collector.process.md b/docs/collector.process.md index 51109d9a..e2d9a6a9 100644 --- a/docs/collector.process.md +++ b/docs/collector.process.md @@ -2,16 +2,23 @@ The process collector exposes metrics about processes. -||| --|- -Metric name prefix | `process` -Data source | Perflib -Counters | `Process` -Enabled by default? | No +Note, on Windows Server 2022, the `Process` counter set is disabled by default. To enable it, run the following command in an elevated PowerShell session: + +```powershell +lodctr.exe /E:Lsa +lodctr.exe /E:PerfProc +lodctr.exe /R +``` + +| | | +|---------------------|-----------| +| Metric name prefix | `process` | +| Data source | Perflib | +| Counters | `Process` | +| Enabled by default? | No | ## Flags -<<<<<<< HEAD ### `--collector.process.include` Regexp of processes to include. Process name must both match `include` and not @@ -30,6 +37,12 @@ Enables IIS process name queries. IIS process names are combined with their app Disabled by default, and can be enabled with `--collector.process.iis=true`. +### `--collector.process.report-owner` + +Enables reporting of the process owner. This is a potentially expensive operation. + +Disabled by default, and can be enabled with `--collector.process.report-owner`. + ### Example To match all firefox processes: `--collector.process.include="firefox.*"`. Note that multiple processes with the same name will be disambiguated by @@ -61,23 +74,23 @@ w3wp_Test ## Metrics -Name | Description | Type | Labels ------|-------------|------|------- -`windows_process_start_time` | Time of process start | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_cpu_time_total` | Returns elapsed time that all of the threads of this process used the processor to execute instructions by mode (privileged, user). An instruction is the basic unit of execution in a computer, a thread is the object that executes instructions, and a process is the object created when a program is run. Code executed to handle some hardware interrupts and trap conditions is included in this count. | counter | `process`, `process_id`, `creating_process_id` -`windows_process_handles` | Total number of handles the process has open. This number is the sum of the handles currently open by each thread in the process. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_io_bytes_total` | Bytes issued to I/O operations in different modes (read, write, other). This property counts all I/O activity generated by the process to include file, network, and device I/Os. Read and write mode includes data operations; other mode includes those that do not involve data, such as control operations. | counter | `process`, `process_id`, `creating_process_id` -`windows_process_io_operations_total` | I/O operations issued in different modes (read, write, other). This property counts all I/O activity generated by the process to include file, network, and device I/Os. Read and write mode includes data operations; other mode includes those that do not involve data, such as control operations. | counter | `process`, `process_id`, `creating_process_id` -`windows_process_page_faults_total` | Page faults by the threads executing in this process. A page fault occurs when a thread refers to a virtual memory page that is not in its working set in main memory. This can cause the page not to be fetched from disk if it is on the standby list and hence already in main memory, or if it is in use by another process with which the page is shared. | counter | `process`, `process_id`, `creating_process_id` -`windows_process_page_file_bytes` | Current number of bytes this process has used in the paging file(s). Paging files are used to store pages of memory used by the process that are not contained in other files. Paging files are shared by all processes, and lack of space in paging files can prevent other processes from allocating memory. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_pool_bytes` | Pool Bytes is the last observed number of bytes in the paged or nonpaged pool. The nonpaged pool is an area of system memory (physical memory used by the operating system) for objects that cannot be written to disk, but must remain in physical memory as long as they are allocated. The paged pool is an area of system memory (physical memory used by the operating system) for objects that can be written to disk when they are not being used. Nonpaged pool bytes is calculated differently than paged pool bytes, so it might not equal the total of paged pool bytes. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_priority_base` | Current base priority of this process. Threads within a process can raise and lower their own base priority relative to the process base priority of the process. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_private_bytes` | Current number of bytes this process has allocated that cannot be shared with other processes. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_threads` | Number of threads currently active in this process. An instruction is the basic unit of execution in a processor, and a thread is the object that executes instructions. Every running process has at least one thread. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_virtual_bytes` | Current size, in bytes, of the virtual address space that the process is using. Use of virtual address space does not necessarily imply corresponding use of either disk or main memory pages. Virtual space is finite and, by using too much, the process can limit its ability to load libraries. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_working_set_private_bytes` | Size of the working set, in bytes, that is use for this process only and not shared nor shareable by other processes. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_working_set_peak_bytes` | Maximum size, in bytes, of the Working Set of this process at any point in time. The Working Set is the set of memory pages touched recently by the threads in the process. If free memory in the computer is above a threshold, pages are left in the Working Set of a process even if they are not in use. When free memory falls below a threshold, pages are trimmed from Working Sets. If they are needed they will then be soft-faulted back into the Working Set before they leave main memory. | gauge | `process`, `process_id`, `creating_process_id` -`windows_process_working_set_bytes` | Maximum number of bytes in the working set of this process at any point in time. The working set is the set of memory pages touched recently by the threads in the process. If free memory in the computer is above a threshold, pages are left in the working set of a process even if they are not in use. When free memory falls below a threshold, pages are trimmed from working sets. If they are needed, they are then soft-faulted back into the working set before they leave main memory. | gauge | `process`, `process_id`, `creating_process_id` +| Name | Description | Type | Labels | +|---------------------------------------------|-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------|---------|-----------------------------------------------------------------| +| `windows_process_start_time` | Time of process start | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_cpu_time_total` | Returns elapsed time that all of the threads of this process used the processor to execute instructions by mode (privileged, user). An instruction is the basic unit of execution in a computer, a thread is the object that executes instructions, and a process is the object created when a program is run. Code executed to handle some hardware interrupts and trap conditions is included in this count. | counter | `process`, `process_id`, `creating_process_id`, `owner`, `mode` | +| `windows_process_handles` | Total number of handles the process has open. This number is the sum of the handles currently open by each thread in the process. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_io_bytes_total` | Bytes issued to I/O operations in different modes (read, write, other). This property counts all I/O activity generated by the process to include file, network, and device I/Os. Read and write mode includes data operations; other mode includes those that do not involve data, such as control operations. | counter | `process`, `process_id`, `creating_process_id`, `owner`, `mode` | +| `windows_process_io_operations_total` | I/O operations issued in different modes (read, write, other). This property counts all I/O activity generated by the process to include file, network, and device I/Os. Read and write mode includes data operations; other mode includes those that do not involve data, such as control operations. | counter | `process`, `process_id`, `creating_process_id`, `owner`, `mode` | +| `windows_process_page_faults_total` | Page faults by the threads executing in this process. A page fault occurs when a thread refers to a virtual memory page that is not in its working set in main memory. This can cause the page not to be fetched from disk if it is on the standby list and hence already in main memory, or if it is in use by another process with which the page is shared. | counter | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_page_file_bytes` | Current number of bytes this process has used in the paging file(s). Paging files are used to store pages of memory used by the process that are not contained in other files. Paging files are shared by all processes, and lack of space in paging files can prevent other processes from allocating memory. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_pool_bytes` | Pool Bytes is the last observed number of bytes in the paged or nonpaged pool. The nonpaged pool is an area of system memory (physical memory used by the operating system) for objects that cannot be written to disk, but must remain in physical memory as long as they are allocated. The paged pool is an area of system memory (physical memory used by the operating system) for objects that can be written to disk when they are not being used. Nonpaged pool bytes is calculated differently than paged pool bytes, so it might not equal the total of paged pool bytes. | gauge | `process`, `process_id`, `creating_process_id`, `owner`, `pool` | +| `windows_process_priority_base` | Current base priority of this process. Threads within a process can raise and lower their own base priority relative to the process base priority of the process. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_private_bytes` | Current number of bytes this process has allocated that cannot be shared with other processes. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_threads` | Number of threads currently active in this process. An instruction is the basic unit of execution in a processor, and a thread is the object that executes instructions. Every running process has at least one thread. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_virtual_bytes` | Current size, in bytes, of the virtual address space that the process is using. Use of virtual address space does not necessarily imply corresponding use of either disk or main memory pages. Virtual space is finite and, by using too much, the process can limit its ability to load libraries. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_working_set_private_bytes` | Size of the working set, in bytes, that is use for this process only and not shared nor shareable by other processes. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_working_set_peak_bytes` | Maximum size, in bytes, of the Working Set of this process at any point in time. The Working Set is the set of memory pages touched recently by the threads in the process. If free memory in the computer is above a threshold, pages are left in the Working Set of a process even if they are not in use. When free memory falls below a threshold, pages are trimmed from Working Sets. If they are needed they will then be soft-faulted back into the Working Set before they leave main memory. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | +| `windows_process_working_set_bytes` | Maximum number of bytes in the working set of this process at any point in time. The working set is the set of memory pages touched recently by the threads in the process. If free memory in the computer is above a threshold, pages are left in the working set of a process even if they are not in use. When free memory falls below a threshold, pages are trimmed from working sets. If they are needed, they are then soft-faulted back into the working set before they leave main memory. | gauge | `process`, `process_id`, `creating_process_id`, `owner` | ### Example metric _This collector does not yet have explained examples, we would appreciate your help adding them!_ diff --git a/pkg/collector/process/process.go b/pkg/collector/process/process.go index 8a15ae5a..ee9864ae 100644 --- a/pkg/collector/process/process.go +++ b/pkg/collector/process/process.go @@ -3,10 +3,13 @@ package process import ( + "errors" "fmt" + "golang.org/x/sys/windows" "regexp" "strconv" "strings" + "syscall" "github.com/alecthomas/kingpin/v2" "github.com/go-kit/log" @@ -23,18 +26,21 @@ const ( FlagProcessExclude = "collector.process.exclude" FlagProcessInclude = "collector.process.include" FlagEnableWorkerProcess = "collector.process.iis" + FlagEnableReportOwner = "collector.process.report-owner" ) type Config struct { ProcessInclude string `yaml:"process_include"` ProcessExclude string `yaml:"process_exclude"` EnableWorkerProcess bool `yaml:"enable_iis_worker_process"` + EnableReportOwner bool `yaml:"enable_report_owner"` } var ConfigDefaults = Config{ ProcessInclude: ".+", ProcessExclude: "", EnableWorkerProcess: false, + EnableReportOwner: false, } type collector struct { @@ -44,6 +50,7 @@ type collector struct { processExclude *string enableWorkerProcess *bool + enableReportOwner *bool StartTime *prometheus.Desc CPUTimeTotal *prometheus.Desc @@ -63,6 +70,8 @@ type collector struct { processIncludePattern *regexp.Regexp processExcludePattern *regexp.Regexp + + lookupCache map[string]string } func New(logger log.Logger, config *Config) types.Collector { @@ -95,6 +104,11 @@ func NewWithFlags(app *kingpin.Application) types.Collector { FlagEnableWorkerProcess, "Enable IIS worker process name queries. May cause the collector to leak memory.", ).Default("false").Bool(), + + enableReportOwner: app.Flag( + FlagEnableReportOwner, + "Enable reporting of process owner.", + ).Default("false").Bool(), } return c } @@ -116,97 +130,104 @@ func (c *collector) Build() error { _ = level.Warn(c.logger).Log("msg", "No filters specified for process collector. This will generate a very large number of metrics!") } + commonLabels := make([]string, 0) + if *c.enableReportOwner { + commonLabels = []string{"owner"} + } + c.StartTime = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "start_time"), "Time of process start.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.CPUTimeTotal = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "cpu_time_total"), "Returns elapsed time that all of the threads of this process used the processor to execute instructions by mode (privileged, user).", - []string{"process", "process_id", "creating_process_id", "mode"}, + append(commonLabels, "process", "process_id", "creating_process_id", "mode"), nil, ) c.HandleCount = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "handles"), "Total number of handles the process has open. This number is the sum of the handles currently open by each thread in the process.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.IOBytesTotal = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "io_bytes_total"), "Bytes issued to I/O operations in different modes (read, write, other).", - []string{"process", "process_id", "creating_process_id", "mode"}, + append(commonLabels, "process", "process_id", "creating_process_id", "mode"), nil, ) c.IOOperationsTotal = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "io_operations_total"), "I/O operations issued in different modes (read, write, other).", - []string{"process", "process_id", "creating_process_id", "mode"}, + append(commonLabels, "process", "process_id", "creating_process_id", "mode"), nil, ) c.PageFaultsTotal = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "page_faults_total"), "Page faults by the threads executing in this process.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.PageFileBytes = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "page_file_bytes"), "Current number of bytes this process has used in the paging file(s).", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.PoolBytes = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "pool_bytes"), "Pool Bytes is the last observed number of bytes in the paged or nonpaged pool.", - []string{"process", "process_id", "creating_process_id", "pool"}, + append(commonLabels, "process", "process_id", "creating_process_id", "pool"), nil, ) c.PriorityBase = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "priority_base"), "Current base priority of this process. Threads within a process can raise and lower their own base priority relative to the process base priority of the process.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.PrivateBytes = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "private_bytes"), "Current number of bytes this process has allocated that cannot be shared with other processes.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.ThreadCount = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "threads"), "Number of threads currently active in this process.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.VirtualBytes = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "virtual_bytes"), "Current size, in bytes, of the virtual address space that the process is using.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.WorkingSetPrivate = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "working_set_private_bytes"), "Size of the working set, in bytes, that is use for this process only and not shared nor shareable by other processes.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.WorkingSetPeak = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "working_set_peak_bytes"), "Maximum size, in bytes, of the Working Set of this process at any point in time. The Working Set is the set of memory pages touched recently by the threads in the process.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) c.WorkingSet = prometheus.NewDesc( prometheus.BuildFQName(types.Namespace, Name, "working_set_bytes"), "Maximum number of bytes in the working set of this process at any point in time. The working set is the set of memory pages touched recently by the threads in the process.", - []string{"process", "process_id", "creating_process_id"}, + append(commonLabels, "process", "process_id", "creating_process_id"), nil, ) + c.lookupCache = make(map[string]string) + var err error c.processIncludePattern, err = regexp.Compile(fmt.Sprintf("^(?:%s)$", *c.processInclude)) @@ -274,6 +295,8 @@ func (c *collector) Collect(ctx *types.ScrapeContext, ch chan<- prometheus.Metri } } + var owner string + for _, process := range data { if process.Name == "_Total" || c.processExcludePattern.MatchString(process.Name) || @@ -294,205 +317,204 @@ func (c *collector) Collect(ctx *types.ScrapeContext, ch chan<- prometheus.Metri } } + labels := make([]string, 0, 4) + + if *c.enableReportOwner { + owner, err = c.getProcessOwner(int(process.IDProcess)) + if err != nil { + owner = "unknown" + } + + labels = []string{owner} + } + + labels = append(labels, processName, pid, cpid) + ch <- prometheus.MustNewConstMetric( c.StartTime, prometheus.GaugeValue, process.ElapsedTime, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.HandleCount, prometheus.GaugeValue, process.HandleCount, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.CPUTimeTotal, prometheus.CounterValue, process.PercentPrivilegedTime, - processName, - pid, - cpid, - "privileged", + append(labels, "privileged")..., ) ch <- prometheus.MustNewConstMetric( c.CPUTimeTotal, prometheus.CounterValue, process.PercentUserTime, - processName, - pid, - cpid, - "user", + append(labels, "user")..., ) ch <- prometheus.MustNewConstMetric( c.IOBytesTotal, prometheus.CounterValue, process.IOOtherBytesPerSec, - processName, - pid, - cpid, - "other", + append(labels, "other")..., ) ch <- prometheus.MustNewConstMetric( c.IOOperationsTotal, prometheus.CounterValue, process.IOOtherOperationsPerSec, - processName, - pid, - cpid, - "other", + append(labels, "other")..., ) ch <- prometheus.MustNewConstMetric( c.IOBytesTotal, prometheus.CounterValue, process.IOReadBytesPerSec, - processName, - pid, - cpid, - "read", + append(labels, "read")..., ) ch <- prometheus.MustNewConstMetric( c.IOOperationsTotal, prometheus.CounterValue, process.IOReadOperationsPerSec, - processName, - pid, - cpid, - "read", + append(labels, "read")..., ) ch <- prometheus.MustNewConstMetric( c.IOBytesTotal, prometheus.CounterValue, process.IOWriteBytesPerSec, - processName, - pid, - cpid, - "write", + append(labels, "write")..., ) ch <- prometheus.MustNewConstMetric( c.IOOperationsTotal, prometheus.CounterValue, process.IOWriteOperationsPerSec, - processName, - pid, - cpid, - "write", + append(labels, "write")..., ) ch <- prometheus.MustNewConstMetric( c.PageFaultsTotal, prometheus.CounterValue, process.PageFaultsPerSec, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.PageFileBytes, prometheus.GaugeValue, process.PageFileBytes, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.PoolBytes, prometheus.GaugeValue, process.PoolNonpagedBytes, - processName, - pid, - cpid, - "nonpaged", + append(labels, "nonpaged")..., ) ch <- prometheus.MustNewConstMetric( c.PoolBytes, prometheus.GaugeValue, process.PoolPagedBytes, - processName, - pid, - cpid, - "paged", + append(labels, "paged")..., ) ch <- prometheus.MustNewConstMetric( c.PriorityBase, prometheus.GaugeValue, process.PriorityBase, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.PrivateBytes, prometheus.GaugeValue, process.PrivateBytes, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.ThreadCount, prometheus.GaugeValue, process.ThreadCount, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.VirtualBytes, prometheus.GaugeValue, process.VirtualBytes, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.WorkingSetPrivate, prometheus.GaugeValue, process.WorkingSetPrivate, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.WorkingSetPeak, prometheus.GaugeValue, process.WorkingSetPeak, - processName, - pid, - cpid, + labels..., ) ch <- prometheus.MustNewConstMetric( c.WorkingSet, prometheus.GaugeValue, process.WorkingSet, - processName, - pid, - cpid, + labels..., ) } return nil } + +// ref: https://github.com/microsoft/hcsshim/blob/8beabacfc2d21767a07c20f8dd5f9f3932dbf305/internal/uvm/stats.go#L25 +func (c *collector) getProcessOwner(pid int) (string, error) { + p, err := windows.OpenProcess(windows.PROCESS_QUERY_LIMITED_INFORMATION, false, uint32(pid)) + if errors.Is(err, syscall.Errno(0x57)) { // invalid parameter, for PIDs that don't exist + return "", errors.New("process not found") + } + + if err != nil { + return "", fmt.Errorf("OpenProcess: %T %w", err, err) + } + + defer windows.Close(p) + + var tok windows.Token + if err = windows.OpenProcessToken(p, windows.TOKEN_QUERY, &tok); err != nil { + return "", fmt.Errorf("OpenProcessToken: %w", err) + } + + tokenUser, err := tok.GetTokenUser() + if err != nil { + return "", fmt.Errorf("GetTokenUser: %w", err) + } + + sid := tokenUser.User.Sid.String() + if owner, ok := c.lookupCache[sid]; ok { + return owner, nil + } + + account, domain, _, err := tokenUser.User.Sid.LookupAccount("") + if err != nil { + c.lookupCache[sid] = sid + } else { + c.lookupCache[sid] = fmt.Sprintf(`%s\%s`, account, domain) + } + + return c.lookupCache[sid], nil +}