From 79506d0661c76a96946ca4061bd8a47ccfd055a7 Mon Sep 17 00:00:00 2001 From: Bob Roldan Date: Thu, 5 Nov 2020 12:33:30 -0500 Subject: [PATCH] DBZ-2659 Restore content from partials files to main `monitoring.adoc` file. --- .../ROOT/pages/operations/monitoring.adoc | 297 +++++++++++++++++- 1 file changed, 292 insertions(+), 5 deletions(-) diff --git a/documentation/modules/ROOT/pages/operations/monitoring.adoc b/documentation/modules/ROOT/pages/operations/monitoring.adoc index b0fe0c907..33894fa98 100644 --- a/documentation/modules/ROOT/pages/operations/monitoring.adoc +++ b/documentation/modules/ROOT/pages/operations/monitoring.adoc @@ -1,3 +1,8 @@ + +// Category: debezium-using +// Type: assembly +// ModuleID: monitoring-debezium + [id="monitoring-debezium"] = Monitoring {prodname} @@ -11,7 +16,9 @@ toc::[] You can use the JMX metrics provided by link:https://zookeeper.apache.org/doc/r3.1.2/zookeeperJMX.html[Zookeeper] and link:http://docs.confluent.io/3.0.0/kafka/monitoring.html[Kafka] to monitor {prodname}. To use these metrics, you must enable them when you start the Zookeeper, Kafka, and Kafka Connect services. Enabling JMX involves setting the correct environment variables. -The environment variables that you must set depends on whether you are running Zookeeper, Kafka, and Kafka Connect in a local installation or in Docker containers. +ifdef::community[] +The environment variables that you must set depend on whether you are running Zookeeper, Kafka, and Kafka Connect in a local installation or in Docker containers. +endif::community[] [NOTE] ==== @@ -19,10 +26,290 @@ If you are running multiple services on the same machine, be sure to use distinct JMX ports for each service. ==== -include::{partialsdir}/modules/monitoring/con-metrics-monitoring-connectors.adoc[leveloffset=+1] -include::{partialsdir}/assemblies/monitoring/assembly-enabling-jmx-local-installations.adoc[leveloffset=+1] +// Category: debezium-using +// Type: concept +// ModuleID: debezium-metrics-monitoring-connectors +[id="metrics-monitoring-connectors"] +== Metrics for monitoring {prodname} connectors -include::{partialsdir}/assemblies/monitoring/assembly-enabling-jmx-docker.adoc[leveloffset=+1] +In addition to the built-in support for JMX metrics in Kafka, Zookeeper, and Kafka Connect, +each connector provides additional metrics that you can use to monitor their activities. -include::{partialsdir}/modules/monitoring/con-using-prometheus-grafana.adoc[leveloffset=+1] +* {link-prefix}:{link-mysql-connector}#mysql-monitoring[MySQL connector metrics] +* {link-prefix}:{link-mongodb-connector}#mongodb-monitoring[MongoDB connector metrics] +* {link-prefix}:{link-postgresql-connector}#postgresql-monitoring[PosgreSQL connector metrics] +* {link-prefix}:{link-sqlserver-connector}#sqlserver-monitoring[SQL Server connector metrics] +* {link-prefix}:{link-db2-connector}#db2-monitoring[Db2 connector metrics] +ifdef::community[] +* {link-prefix}:{link-oracle-connector}#oracle-monitoring[Oracle connector metrics] +* {link-prefix}:{link-cassandra-connector}#cassandra-monitoring[Cassandra connector metrics] +endif::community[] + + +// Category: debezium-using +// Type: assembly +// ModuleID: enabling-jmx-in-local-installations +[id="enabling-jmx-local-installation"] +== Enabling JMX in local installations + +With Zookeeper, Kafka, and Kafka Connect, +you enable JMX by setting the appropriate environment variables when you start each service. + +// Category: debezium-using +// Type: reference +// ModuleID: debezium-zookeeper-jmx-environment-variables] +[id="zookeeper-jmx-environment-variables"] +=== Zookeeper JMX environment variables + +Zookeeper has built-in support for JMX. +When running Zookeeper using a local installation, +the `zkServer.sh` script recognizes the following environment variables: + +`JMXPORT`:: +Enables JMX and specifies the port number that will be used for JMX. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.port=$JMXPORT`. + +`JMXAUTH`:: +Whether JMX clients must use password authentication when connecting. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.authenticate=$JMXAUTH`. + +`JMXSSL`:: +Whether JMX clients connect using SSL/TLS. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.ssl=$JMXSSL`. + +`JMXLOG4J`:: +Whether the Log4J JMX MBeans should be disabled. +Must be either `true` (default) or `false`. +The default is `true`. +The value is used to specify the JVM parameter `-Dzookeeper.jmx.log4j.disable=$JMXLOG4J`. + + +// Category: debezium-using +// Type: reference +// ModuleID: debezium-kafka-jmx-environment-variables"] +[id="kafka-jmx-environment-variables"] +=== Kafka JMX environment variables + +When running Kafka using a local installation, +the `kafka-server-start.sh` script recognizes the following environment variables: + +`JMX_PORT`:: +Enables JMX and specifies the port number that will be used for JMX. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.port=$JMX_PORT`. + +`KAFKA_JMX_OPTS`:: +The JMX options, which are passed directly to the JVM during startup. +The default options are: ++ +** `-Dcom.sun.management.jmxremote` +** `-Dcom.sun.management.jmxremote.authenticate=false` +** `-Dcom.sun.management.jmxremote.ssl=false` + + +// Category: debezium-using +// Type: reference +// ModuleID: debezium-kafka-connect-jmx-environment-variables"] +[id="kafka-connect-jmx-environment-variables"] +=== Kafka Connect JMX environment variables + +When running Kafka using a local installation, +the `connect-distributed.sh` script recognizes the following environment variables: + +`JMX_PORT`:: +Enables JMX and specifies the port number that will be used for JMX. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.port=$JMX_PORT`. + +`KAFKA_JMX_OPTS`:: +The JMX options, which are passed directly to the JVM during startup. +The default options are: ++ +** `-Dcom.sun.management.jmxremote` +** `-Dcom.sun.management.jmxremote.authenticate=false` +** `-Dcom.sun.management.jmxremote.ssl=false` + +ifdef::product[] +// Category: debezium-using +// Type: concept +// ModuleID: monitoring-debezium-on-openshift"] +== Monitoring {prodname} on OpenShift + +If you are using {prodname} on OpenShift, you can obtain JMX metrics by opening a JMX port on `9999`. +For more information, see link:{LinkStreamsOpenShift}#assembly-jmx-options-deployment-configuration-kafka[JMX Options]. + +In addition, you can use Prometheus and Grafana to monitor the JMX metrics. +For more information, see link:{LinkStreamsOpenShift}/#assembly-metrics-setup-str[Introducing Metrics]. + +endif::product[] + +ifdef::community[] +[id="enabling-jmx-docker"] +=== Enabling JMX in Docker + +If you are running Zookeeper, Kafka, and Kafka Connect in Docker containers, +enabling JMX requires several additional environment variables that are not typically needed when running on a local machine. +This is because the JVM requires the host name to which it will advertise itself to JMX clients. +Thus, {prodname}'s Docker images for Zookeeper, Kafka, and Kafka Connect use several environment variables to enable and configure JMX. +Most of the environment variables are the same for all of the images, +but there are some minor differences. + + +[id="zookeeper-jmx-environment-variables-docker"] +=== Zookeeper JMX environment variables in Docker + +The `debezium/zookeeper` image recognizes the following JMX-related environment variables: + +`JMXPORT` (Required):: +The port number that will be used for JMX. +The value is used to specify the following JVM parameters: ++ +** `-Dcom.sun.management.jmxremote.port=$JMXPORT` +** `-Dcom.sun.management.jmxremote.rmi.port=$JMXPORT` + +`JMXHOST` (Required):: +The IP address or resolvable host name of the Docker host, +which JMX uses to construct a URL sent to the JMX client. +A value of `localhost` or `127.0.0.1` will not work. +Typically, `0.0.0.0` can be used. +The value is used to specify the JVM parameter `-Djava.rmi.server.hostname=$JMXHOST`. + +`JMXAUTH`:: +Whether JMX clients must use password authentication when connecting. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.authenticate=$JMXAUTH`. + +`JMXSSL`:: +Whether JMX clients connect using SSL/TLS. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.ssl=$JMXSSL`. + +`JMXLOG4J`:: +Whether the Log4J JMX MBeans should be disabled. +Must be either `true` or `false`. +The default is `true`. +The value is used to specify the JVM parameter `-Dzookeeper.jmx.log4j.disable=$JMXLOG4J`. + +.Enabling JMX in a Zookeeper Docker container +==== +The following example Docker command starts a container using the `debezium/zookeeper` image with values for the `JMXPORT` and `JMXHOST` environment variables, +and maps the Docker host's port 9010 to the container's JMX port: + +[source,shell,options="nowrap"] +---- +$ docker run -it --rm --name zookeeper -p 2181:2181 -p 2888:2888 -p 3888:3888 -p 9010:9010 -e JMXPORT=9010 -e JMXHOST=10.0.1.10 debezium/zookeeper:latest +---- +==== + +[id="kafka-jmx-environment-variables-docker"] +=== Kafka JMX environment variables in Docker + +The `debezium/kafka` image recognizes the following JMX-related environment variables: + +`JMXPORT` (Required):: +The port number that will be used for JMX. +The value is used to specify the following JVM parameters: ++ +** `-Dcom.sun.management.jmxremote.port=$JMXPORT` +** `-Dcom.sun.management.jmxremote.rmi.port=$JMXPORT` + +`JMXHOST` (Required):: +The IP address or resolvable host name of the Docker host, +which JMX uses to construct a URL sent to the JMX client. +A value of `localhost` or `127.0.0.1` will not work. +Typically, `0.0.0.0` can be used. +The value is used to specify the JVM parameter `-Djava.rmi.server.hostname=$JMXHOST`. + +`JMXAUTH`:: +Whether JMX clients must use password authentication when connecting. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.authenticate=$JMXAUTH`. + +`JMXSSL`:: +Whether JMX clients connect using SSL/TLS. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.ssl=$JMXSSL`. + +.Enabling JMX in a Kafka Docker container +==== +The following example Docker command starts a container using the `debezium/kafka` image with values for the `JMXPORT` and `HOST_NAME` environment variables, +and maps the Docker host's port 9011 to the container's JMX port: + +[source,shell,options="nowrap"] +---- +$ docker run -it --rm --name kafka -p 9092:9092 -p 9011:9011 -e JMXPORT=9011 -e JMXHOST=10.0.1.10 --link zookeeper:zookeeper debezium/kafka:latest +---- +==== + +[id="kafka-connect-jmx-environment-variables-docker"] +=== Kafka Connect JMX environment variables in Docker + +The `debezium/connect` image recognizes the following JMX-related environment variables: + +`JMXPORT` (Required):: +The port number that will be used for JMX. +The value is used to specify the following JVM parameters: ++ +** `-Dcom.sun.management.jmxremote.port=$JMXPORT` +** `-Dcom.sun.management.jmxremote.rmi.port=$JMXPORT` + +`JMXHOST` (Required):: +The IP address or resolvable host name of the Docker host, +which JMX uses to construct a URL sent to the JMX client. +A value of `localhost` or `127.0.0.1` will not work. +Typically, `0.0.0.0` can be used. +The value is used to specify the JVM parameter `-Djava.rmi.server.hostname=$JMXHOST`. + +`JMXAUTH`:: +Whether JMX clients must use password authentication when connecting. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.authenticate=$JMXAUTH`. + +`JMXSSL`:: +Whether JMX clients connect using SSL/TLS. +Must be either `true` or `false`. +The default is `false`. +The value is used to specify the JVM parameter `-Dcom.sun.management.jmxremote.ssl=$JMXSSL`. + +.Enabling JMX in a Kafka Connect Docker container +==== +The following example Docker command starts a container using the `debezium/connect` image with values for the `JMXPORT` and `JMXHOST` environment variables, +and maps the Docker host's port 9012 to the container's JMX port: + +[source,shell,options="nowrap"] +---- +$ docker run -it --rm --name connect \ + -p 8083:8083 -p 9012:9012 \ + -e JMXPORT=9012 -e JMXHOST=10.0.1.10 \ + -e GROUP_ID=1 \ + -e CONFIG_STORAGE_TOPIC=my_connect_configs \ + -e OFFSET_STORAGE_TOPIC=my_connect_offsets \ + -e STATUS_STORAGE_TOPIC=my_connect_statuses \ + --link zookeeper:zookeeper \ + --link kafka:kafka \ + --link mysql:mysql \ + debezium/connect:latest +---- +==== + +[id="using-prometheus-grafana"] +== Using Prometheus and Grafana + +The metrics exposed by {prodname} and Kafka can be exported and displayed with https://prometheus.io/[Prometheus] and https://grafana.com/[Grafana]. +You can find an example for the required configuration and example dashboards for different connectors in the https://github.com/debezium/debezium-examples/tree/master/monitoring[Debezium examples repository]. + +[NOTE] +==== +These dashboards are not part of {prodname} itself and are maintained on a best-effort basis. +==== + + +endif::community[]