Skip to main content
Skip table of contents

KPIs to be monitored

Disk

  • for Persistence should be used less than 75% at any time.

  • disk for WEBAPP should be used less then 85% at any time.

Memory

  • there should be always 1Gb memory free

CPU

  • the load on the server should be less then 75% at any time.

Services should be up and running

  • this can be checked by using script /etc/veridiumid/scripts/check_services.sh every minute.

Application

Metrics

Probe

websec

/websecadmin/rest/health/metrics/all

/websec/rest/health

websecadmin

/websecadmin/rest/health/metrics/all

websecadmin/rest/health

adservice

/websecadmin/rest/health/metrics/all

shibboleth

/idp/profile/admin/metrics

/idp/profile/status

Application - Invoke websecadmin API /websecadmin/rest/health/metrics/all on each application server.

The response should have the following parameters, and they should be within limits:

  1. cassandra.response.time value less than 50 (if cassandra is down, this field will be missing)

  2. opa.health.check.response.time → status true

  3. zookeeper.response.time value less then 50

  4. if response code for this API is 503, the service websecadmin is unavailable.

  5. this Api also presents for each application, it’s status true/false.

In below example all the services have status TRUE, so they are responding.

An example of a response from the endpoint above is the following:

CODE 
{
  "list": [
    {
      "name": "websecadmin-Demo1-10.64.0.196",
      "status": true,
      "metrics": [
        {
          "name": "cassandra.response.time",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "Cassandra response time in ms",
          "unit": "MS",
          "value": 1,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "target-hosts": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "kafka.response.time",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "Kafka response time",
          "unit": "MS",
          "value": 3,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "target-hosts": "10.64.0.196:9095",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "system.jvm.memory.free",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "System jvm free memory in mb",
          "unit": "MB",
          "value": 242,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "system.jvm.memory.max",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "System jvm max memory in mb",
          "unit": "MB",
          "value": 1007,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "system.jvm.memory.total",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "System jvm total memory in mb",
          "unit": "MB",
          "value": 1007,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "system.jvm.memory.used",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "System jvm used memory in mb",
          "unit": "MB",
          "value": 765,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "zookeeper.response.time",
          "componentName": "websecadmin-Demo1-10.64.0.196",
          "serviceName": "websecadmin",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522835932,
          "description": "Zookeper response time",
          "unit": "MS",
          "value": 1,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "target-hosts": "10.64.0.196:2181/veridiumid/7.1.34",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        }
      ],
      "error": {
        "errorCode": 0,
        "errorDescription": ""
      }
    },
    {
      "name": "websec-Demo1-10.64.0.196",
      "status": true,
      "metrics": [
        {
          "name": "api-websec-authenticate",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "VeridiumID API Authenticate response time in ms",
          "unit": "MS",
          "value": 1,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "api-websec-register",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "VeridiumID API Register response time in ms",
          "unit": "MS",
          "value": 1,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "cassandra.response.time",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "Cassandra response time in ms",
          "unit": "MS",
          "value": 1,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "target-hosts": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "crypto.sign.response",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836874,
          "description": "HSM Sign response time in ms",
          "unit": "MS",
          "value": 154,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "kafka.response.time",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "Kafka response time",
          "unit": "MS",
          "value": 3,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "target-hosts": "10.64.0.196:9095",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "opa.health.check.response.time",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "OPA health check response time in ms",
          "unit": "MS",
          "value": 2,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "system.jvm.memory.free",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "System jvm free memory in mb",
          "unit": "MB",
          "value": 660,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "system.jvm.memory.max",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "System jvm max memory in mb",
          "unit": "MB",
          "value": 1954,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "system.jvm.memory.total",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "System jvm total memory in mb",
          "unit": "MB",
          "value": 1954,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "system.jvm.memory.used",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "System jvm used memory in mb",
          "unit": "MB",
          "value": 1294,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true"
          }
        },
        {
          "name": "zookeeper.response.time",
          "componentName": "websec-Demo1-10.64.0.196",
          "serviceName": "websec",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522836897,
          "description": "Zookeper response time",
          "unit": "MS",
          "value": 1,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "target-hosts": "10.64.0.196:2181/veridiumid/7.1.34",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        }
      ],
      "error": {
        "errorCode": 0,
        "errorDescription": ""
      }
    },
    {
      "name": "adservice-Demo1-10.64.0.196",
      "status": true,
      "metrics": [
        {
          "name": "ldap.query.response.time.dev2",
          "componentName": "adservice-Demo1-10.64.0.196",
          "serviceName": "adservice",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522837772,
          "description": "LDAP Query Response time in ms",
          "unit": "MS",
          "value": 55,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        },
        {
          "name": "ldap.query.response.time.devDuplicated",
          "componentName": "adservice-Demo1-10.64.0.196",
          "serviceName": "adservice",
          "serverIdentifier": "Demo1",
          "ip": "10.64.0.196",
          "date": 1685522837772,
          "description": "LDAP Query Response time in ms",
          "unit": "MS",
          "value": 51,
          "tags": {
            "hostname": "Demo1",
            "product-version": "3.3.0",
            "ip": "10.64.0.196",
            "product-build": "7.1.34",
            "built-on": "27/04/2023",
            "enabled": "true",
            "status": "true"
          }
        }
      ],
      "error": {
        "errorCode": 0,
        "errorDescription": ""
      }
    }
  ],
  "error": {
    "errorCode": 0,
    "errorDescription": ""
  }
}

SHIBBOLETH Metrics

CODE 
{
  "version": "3.0.0",
  "gauges": {},
  "counters": {},
  "histograms": {},
  "meters": {},
  "timers": {
    "idp.attribute.resolution": {
      "count": 21,
      "max": 0.018612964000000003,
      "mean": 0.01062199668190866,
      "min": 0.009670108,
      "p50": 0.010938089000000002,
      "p75": 0.010938089000000002,
      "p95": 0.010938089000000002,
      "p98": 0.017655501,
      "p99": 0.017655501,
      "p999": 0.017655501,
      "values": [
        0.009670108,
        0.009744391,
        0.010331370000000001,
        0.010938089000000002,
        0.012251374,
        0.012360427,
        0.012465291000000002,
        0.013221236,
        0.015282553,
        0.015445979,
        0.015546423,
        0.016093163,
        0.016433223,
        0.016960598,
        0.017655501,
        0.018478008,
        0.018612964000000003
      ],
      "stddev": 0.0014430800056764548,
      "m15_rate": 1.410603874572755e-05,
      "m1_rate": 2.204365757112475e-44,
      "m5_rate": 6.478439436580341e-11,
      "mean_rate": 0.0002512594208464394,
      "duration_units": "seconds",
      "rate_units": "calls/second"
    }
  }
}
JavaScript errors detected

Please note, these errors can depend on your browser setup.

If this problem persists, please contact our support.

Contact Support Close