From 72937de200cb22b300970c7080a8a5040737010f Mon Sep 17 00:00:00 2001
From: Killian Murphy <killian.murphy@york.ac.uk>
Date: Wed, 11 Nov 2020 14:57:38 +0000
Subject: Actually rename the QoS pending raider

---
 slurm_pending_on_qos.rb | 52 +++++++++++++++++++++++++++++++++++++++++++++++++
 1 file changed, 52 insertions(+)
 create mode 100644 slurm_pending_on_qos.rb

diff --git a/slurm_pending_on_qos.rb b/slurm_pending_on_qos.rb
new file mode 100644
index 0000000..288719a
--- /dev/null
+++ b/slurm_pending_on_qos.rb
@@ -0,0 +1,52 @@
+# frozen_string_literal: true
+
+require 'date'
+
+# For each Viking partition, report the number of jobs pending for a long time
+# due to QoS reasons.
+class SlurmPendingOnQos
+  def initialize(collector, config)
+    @collector = collector
+    @partition_thresholds = {
+      nodes: 604_800,
+      week: 1_209_600,
+      month: 2_419_200,
+      himem: 604_800,
+      himem_week: 1_209_600,
+      gpu: 604_800,
+      interactive: 900,
+      test: 900,
+      preempt: 2_419_200
+    }
+  end
+
+  def raid
+    @partition_thresholds.each do |partition, threshold|
+      squeue_cmd = [
+        'squeue',
+        '--format="%R,%V"',
+        '--noheader',
+        "--partition=#{partition}",
+        '--state=PENDING'
+      ].join(' ')
+
+      data = `#{squeue_cmd}`.split("\n").grep(/QOS/).map do |row|
+        row.split(',')
+      end
+
+      count = data.count do |columns|
+        (Time.now.to_i - DateTime.parse(columns[1]).to_time.to_i) > threshold
+      end
+
+      @collector.report!(
+        'pending_on_qos',
+        count,
+        help: 'Number of jobs pending beyond a threshold for QoS reasons',
+        type: 'gauge',
+        labels: {
+          partition: partition.to_s
+        }
+      )
+    end
+  end
+end
-- 
cgit v1.2.1