commit d9afb7b38ab1998a2accc499834af00b99d45a2e Author: Chase Dumphord Date: Sat May 16 23:45:48 2026 -0500 Initial CedOps AI Infrastructure NOC project diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..f45705d --- /dev/null +++ b/.gitignore @@ -0,0 +1,11 @@ +.env +*.env +secrets/ +private/ +*.key +*.pem +*.crt +*.log +tokens.txt +credentials.yml +credentials.yaml \ No newline at end of file diff --git a/Prometheus/blackbox-targets-example.yml b/Prometheus/blackbox-targets-example.yml new file mode 100644 index 0000000..e69de29 diff --git a/Prometheus/prometheus-example.yml b/Prometheus/prometheus-example.yml new file mode 100644 index 0000000..e69de29 diff --git a/README.md b/README.md new file mode 100644 index 0000000..e69de29 diff --git a/diagrams/cedops-architecture.mmd b/diagrams/cedops-architecture.mmd new file mode 100644 index 0000000..e69de29 diff --git a/docs/screenshots/dashboard_V1.png b/docs/screenshots/dashboard_V1.png new file mode 100644 index 0000000..7aaba55 Binary files /dev/null and b/docs/screenshots/dashboard_V1.png differ diff --git a/grafana/cedops-ai-infrastructure-noc-v1.json b/grafana/cedops-ai-infrastructure-noc-v1.json new file mode 100644 index 0000000..e863076 --- /dev/null +++ b/grafana/cedops-ai-infrastructure-noc-v1.json @@ -0,0 +1,790 @@ +{ + "annotations": { + "list": [] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 1, + "id": null, + "links": [], + "liveNow": false, + "panels": [ + { + "type": "row", + "title": "CedOps Executive Health", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 1, + "collapsed": false, + "panels": [] + }, + { + "id": 2, + "type": "stat", + "title": "Synthos VPS Up", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "targets": [ + { + "expr": "up{instance=\"synthos-node-1\"}", + "legendFormat": "synthos-node-1", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 3, + "type": "stat", + "title": "Public Services Up", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "targets": [ + { + "expr": "sum(probe_success{job=\"blackbox-http-public\",site=~\"synthos-systems|ceds-home-lab\"})", + "legendFormat": "up", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 3 + }, + { + "color": "green", + "value": 5 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 4, + "type": "stat", + "title": "NOC Health Score", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "targets": [ + { + "expr": "100 * avg(probe_success{job=\"blackbox-http-public\",site=~\"synthos-systems|ceds-home-lab\"})", + "legendFormat": "score", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 80 + }, + { + "color": "green", + "value": 95 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 5, + "type": "stat", + "title": "VPS Uptime", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "targets": [ + { + "expr": "time() - node_boot_time_seconds{instance=\"synthos-node-1\"}", + "legendFormat": "uptime", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "value", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "type": "row", + "title": "Synthos VPS Resource Health", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 6, + "collapsed": false, + "panels": [] + }, + { + "id": 7, + "type": "timeseries", + "title": "VPS CPU Usage %", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "targets": [ + { + "expr": "100 - (avg by(instance)(rate(node_cpu_seconds_total{instance=\"synthos-node-1\",mode=\"idle\"}[5m])) * 100)", + "legendFormat": "{{instance}} CPU", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 8, + "type": "timeseries", + "title": "VPS Memory Usage %", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "targets": [ + { + "expr": "(1 - (node_memory_MemAvailable_bytes{instance=\"synthos-node-1\"} / node_memory_MemTotal_bytes{instance=\"synthos-node-1\"})) * 100", + "legendFormat": "{{instance}} Memory", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 75 + }, + { + "color": "red", + "value": 90 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + } + }, + { + "id": 9, + "type": "stat", + "title": "VPS Disk Used %", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 5, + "w": 8, + "x": 0, + "y": 14 + }, + "targets": [ + { + "expr": "100 - ((node_filesystem_avail_bytes{instance=\"synthos-node-1\",mountpoint=\"/\",fstype!=\"rootfs\"} * 100) / node_filesystem_size_bytes{instance=\"synthos-node-1\",mountpoint=\"/\",fstype!=\"rootfs\"})", + "legendFormat": "disk", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "percent", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 70 + }, + { + "color": "red", + "value": 85 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "auto", + "textMode": "auto", + "colorMode": "background", + "graphMode": "area", + "justifyMode": "auto" + } + }, + { + "id": 10, + "type": "timeseries", + "title": "VPS Network Traffic", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 5, + "w": 16, + "x": 8, + "y": 14 + }, + "targets": [ + { + "expr": "sum by(instance)(rate(node_network_receive_bytes_total{instance=\"synthos-node-1\",device!~\"lo|veth.*|docker.*|br-.*\"}[5m]))", + "legendFormat": "RX", + "refId": "A" + }, + { + "expr": "sum by(instance)(rate(node_network_transmit_bytes_total{instance=\"synthos-node-1\",device!~\"lo|veth.*|docker.*|br-.*\"}[5m]))", + "legendFormat": "TX", + "refId": "B" + } + ], + "fieldConfig": { + "defaults": { + "unit": "Bps", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "list", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + } + }, + { + "type": "row", + "title": "CedOps Public Service Availability", + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 19 + }, + "id": 11, + "collapsed": false, + "panels": [] + }, + { + "id": 12, + "type": "stat", + "title": "Service Status", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 20 + }, + "targets": [ + { + "expr": "probe_success{job=\"blackbox-http-public\",site=~\"synthos-systems|ceds-home-lab\"}", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "mappings": [ + { + "type": "value", + "options": { + "0": { + "text": "DOWN", + "color": "red" + }, + "1": { + "text": "UP", + "color": "green" + } + } + } + ], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "horizontal", + "textMode": "auto", + "colorMode": "background", + "graphMode": "none", + "justifyMode": "auto" + } + }, + { + "id": 13, + "type": "timeseries", + "title": "Public Endpoint Response Time", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 27 + }, + "targets": [ + { + "expr": "probe_duration_seconds{job=\"blackbox-http-public\",site=~\"synthos-systems|ceds-home-lab\"}", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "s", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 1 + }, + { + "color": "red", + "value": 3 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "desc" + } + } + }, + { + "id": 14, + "type": "bargauge", + "title": "HTTP Status Code", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 27 + }, + "targets": [ + { + "expr": "probe_http_status_code{job=\"blackbox-http-public\",site=~\"synthos-systems|ceds-home-lab\"}", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "short", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 200 + }, + { + "color": "yellow", + "value": 300 + }, + { + "color": "red", + "value": 400 + } + ] + } + }, + "overrides": [] + }, + "options": { + "reduceOptions": { + "values": false, + "calcs": [ + "lastNotNull" + ], + "fields": "" + }, + "orientation": "horizontal", + "displayMode": "gradient", + "showUnfilled": true, + "minVizWidth": 0, + "minVizHeight": 10 + } + }, + { + "id": 15, + "type": "timeseries", + "title": "SSL Expiry Days", + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 35 + }, + "targets": [ + { + "expr": "(probe_ssl_earliest_cert_expiry{job=\"blackbox-http-public\",site=~\"synthos-systems|ceds-home-lab\"} - time()) / 86400", + "legendFormat": "{{service}}", + "refId": "A" + } + ], + "fieldConfig": { + "defaults": { + "unit": "d", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 14 + }, + { + "color": "green", + "value": 30 + } + ] + } + }, + "overrides": [] + }, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom" + }, + "tooltip": { + "mode": "multi", + "sort": "asc" + } + } + } + ], + "refresh": "30s", + "schemaVersion": 39, + "style": "dark", + "tags": [ + "cedops", + "synthos", + "prometheus", + "blackbox", + "platform-engineering" + ], + "templating": { + "list": [ + { + "current": {}, + "hide": 0, + "includeAll": false, + "label": "Prometheus", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "refresh": 1, + "regex": "", + "type": "datasource" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "CedOps AI Infrastructure NOC", + "uid": "cedops-ai-infra-noc", + "version": 1, + "weekStart": "" +} \ No newline at end of file