diff --git a/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java b/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java index bc667b1f..a55d2c20 100644 --- a/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java +++ b/hazelcast/src/main/java/org/apache/karaf/cellar/hazelcast/factory/HazelcastConfigurationManager.java @@ -85,17 +85,19 @@ public boolean isUpdated(Map properties) { if (!CellarUtils.collectionEquals(discoveredMemberSet, newDiscoveredMemberSet)) { LOGGER.debug("Hazelcast discoveredMemberSet has been changed from {} to {}", discoveredMemberSet, newDiscoveredMemberSet); discoveredMemberSet = newDiscoveredMemberSet; - for (String discoveredMember:discoveredMemberSet) { - if (discoveredMember != null && !String.valueOf(discoveredMember).equals("null") && !tcpIpConfig.getMembers().contains(discoveredMember)) { - tcpIpConfig.getMembers().add(discoveredMember); - } - } - Iterator iterator = tcpIpConfig.getMembers().iterator(); - while (iterator.hasNext()) { - String member = iterator.next(); - if (!discoveredMemberSet.contains(member)) { - iterator.remove(); - } + if (tcpIpConfig != null) { + for (String discoveredMember:discoveredMemberSet) { + if (discoveredMember != null && !String.valueOf(discoveredMember).equals("null") && !tcpIpConfig.getMembers().contains(discoveredMember)) { + tcpIpConfig.getMembers().add(discoveredMember); + } + } + Iterator iterator = tcpIpConfig.getMembers().iterator(); + while (iterator.hasNext()) { + String member = iterator.next(); + if (!discoveredMemberSet.contains(member)) { + iterator.remove(); + } + } } updated = Boolean.TRUE; } diff --git a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java index c26c5b8e..d6aa6701 100644 --- a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java +++ b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/ConfigKey.java @@ -11,8 +11,10 @@ * GitHub */ public enum ConfigKey { + KUBERNETES_ENABLE_AUTOCONFIGURE("kubernetes.autoConfig"), KUBERNETES_MASTER(Config.KUBERNETES_MASTER_SYSTEM_PROPERTY), KUBERNETES_API_VERSION(Config.KUBERNETES_API_VERSION_SYSTEM_PROPERTY), + KUBERNETES_NAMESPACE(Config.KUBERNETES_NAMESPACE_SYSTEM_PROPERTY), KUBERNETES_TRUST_CERTIFICATES(Config.KUBERNETES_TRUST_CERT_SYSTEM_PROPERTY), KUBERNETES_DISABLE_HOSTNAME_VERIFICATION(Config.KUBERNETES_DISABLE_HOSTNAME_VERIFICATION_SYSTEM_PROPERTY), KUBERNETES_CERTS_CA_FILE(Config.KUBERNETES_CA_CERTIFICATE_FILE_SYSTEM_PROPERTY), diff --git a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java index bed2dcb7..2fd5dbec 100644 --- a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java +++ b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryService.java @@ -13,6 +13,7 @@ */ package org.apache.karaf.cellar.kubernetes; +import io.fabric8.kubernetes.api.model.ObjectMeta; import io.fabric8.kubernetes.api.model.Pod; import io.fabric8.kubernetes.api.model.PodList; import io.fabric8.kubernetes.client.Config; @@ -35,8 +36,11 @@ public class KubernetesDiscoveryService implements DiscoveryService { private static final Logger LOGGER = LoggerFactory.getLogger(KubernetesDiscoveryService.class); + private boolean kubernetesAutoconfigure; + private String kubernetesMaster; private String kubernetesApiVersion; + private String kubernetesNamespace; private boolean kubernetesTrustCertificates; private boolean kubernetesDisableHostnameVerification; private String kubernetesCertsCaFile; @@ -68,31 +72,46 @@ public KubernetesDiscoveryService() { } Config createConfig() { - return new ConfigBuilder() - .withMasterUrl(kubernetesMaster) - .withApiVersion(kubernetesApiVersion) - .withTrustCerts(kubernetesTrustCertificates) - .withDisableHostnameVerification(kubernetesDisableHostnameVerification) - .withCaCertFile(kubernetesCertsCaFile) - .withCaCertData(kubernetesCertsCaData) - .withClientCertFile(kubernetesCertsClientFile) - .withClientCertData(kubernetesCertsClientData) - .withClientKeyFile(kubernetesCertsClientKeyFile) - .withClientKeyData(kubernetesCertsClientKeyData) - .withClientKeyAlgo(kubernetesCertsClientKeyAlgo) - .withClientKeyPassphrase(kubernetesCertsClientKeyPassphrase) - .withUsername(kubernetesAuthBasicUsername) - .withPassword(kubernetesAuthBasicPassword) - .withOauthToken(kubernetesOauthToken) - .withWatchReconnectInterval(kubernetesWatchReconnectInterval) - .withWatchReconnectLimit(kubernetesWatchReconnectLimit) - .withUserAgent(kubernetesUserAgent) - .withTlsVersions(TlsVersion.forJavaName(kubernetesTlsVersion)) - .withTrustStoreFile(kubernetesTruststoreFile) - .withTrustStorePassphrase(kubernetesTruststorePassphrase) - .withKeyStoreFile(kubernetesKeystoreFile) - .withKeyStorePassphrase(kubernetesKeystorePassphrase) - .build(); + TlsVersion[] tlsVersions = {}; + if (kubernetesTlsVersion != null) { + tlsVersions = new TlsVersion[1]; + tlsVersions[0] = TlsVersion.forJavaName(kubernetesTlsVersion); + } + + Config config; + if (kubernetesAutoconfigure) { + config = Config.autoConfigure(null); + config.setTlsVersions(tlsVersions); // avoid NPE in io.fabric8.kubernetes.client.utils.URLUtils.join(URLUtils.java:47) + } else { + config = new ConfigBuilder() + .withMasterUrl(kubernetesMaster) + .withApiVersion(kubernetesApiVersion) + .withNamespace(kubernetesNamespace) + .withTrustCerts(kubernetesTrustCertificates) + .withDisableHostnameVerification(kubernetesDisableHostnameVerification) + .withCaCertFile(kubernetesCertsCaFile) + .withCaCertData(kubernetesCertsCaData) + .withClientCertFile(kubernetesCertsClientFile) + .withClientCertData(kubernetesCertsClientData) + .withClientKeyFile(kubernetesCertsClientKeyFile) + .withClientKeyData(kubernetesCertsClientKeyData) + .withClientKeyAlgo(kubernetesCertsClientKeyAlgo) + .withClientKeyPassphrase(kubernetesCertsClientKeyPassphrase) + .withUsername(kubernetesAuthBasicUsername) + .withPassword(kubernetesAuthBasicPassword) + .withOauthToken(kubernetesOauthToken) + .withWatchReconnectInterval(kubernetesWatchReconnectInterval) + .withWatchReconnectLimit(kubernetesWatchReconnectLimit) + .withUserAgent(kubernetesUserAgent) + .withTlsVersions(tlsVersions) + .withTrustStoreFile(kubernetesTruststoreFile) + .withTrustStorePassphrase(kubernetesTruststorePassphrase) + .withKeyStoreFile(kubernetesKeystoreFile) + .withKeyStorePassphrase(kubernetesKeystorePassphrase) + .build(); + } + + return config; } public void init() { @@ -121,8 +140,8 @@ public Set discoverMembers() { try { PodList podList = kubernetesClient.pods().list(); for (Pod pod : podList.getItems()) { - String value = pod.getMetadata().getLabels().get(kubernetesPodLabelKey); - if (value != null && !value.isEmpty() && value.equals(kubernetesPodLabelValue)) { + String value = getKubernetesPodLabelKey(pod); + if (value != null && !value.isEmpty() && value.equals(kubernetesPodLabelValue) && pod.getStatus().getPodIP() != null) { members.add(pod.getStatus().getPodIP()); } } @@ -132,6 +151,16 @@ public Set discoverMembers() { return members; } + private String getKubernetesPodLabelKey(Pod pod) { + ObjectMeta metadata = pod.getMetadata(); + if (metadata == null) + return null; + Map labels = metadata.getLabels(); + if (labels == null) + return null; + return labels.get(kubernetesPodLabelKey); + } + @Override public void signIn() { // nothing to do for Kubernetes @@ -147,6 +176,16 @@ public void signOut() { // nothing to do for Kubernetes } + public boolean isKubernetesAutoconfigure() { + return kubernetesAutoconfigure; + } + + public void setKubernetesAutoconfigure(String kubernetesAutoconfigure) { + if (kubernetesAutoconfigure != null) { + this.kubernetesAutoconfigure = Boolean.parseBoolean(kubernetesAutoconfigure); + } + } + void setKubernetesClient(KubernetesClient kubernetesClient) { this.kubernetesClient = kubernetesClient; } @@ -183,6 +222,14 @@ public void setKubernetesApiVersion(String kubernetesApiVersion) { this.kubernetesApiVersion = kubernetesApiVersion; } + public String getKubernetesNamespace() { + return kubernetesNamespace; + } + + public void setKubernetesNamespace(String kubernetesNamespace) { + this.kubernetesNamespace = kubernetesNamespace; + } + public boolean isKubernetesTrustCertificates() { return kubernetesTrustCertificates; } diff --git a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java index 44b73dc7..628233ef 100644 --- a/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java +++ b/kubernetes/src/main/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactory.java @@ -13,6 +13,7 @@ */ package org.apache.karaf.cellar.kubernetes; +import io.fabric8.kubernetes.client.Config; import org.apache.karaf.cellar.core.discovery.DiscoveryService; import org.osgi.framework.BundleContext; import org.osgi.framework.ServiceRegistration; @@ -40,9 +41,11 @@ import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_CERTS_CLIENT_KEY_FILE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_CERTS_CLIENT_KEY_PASSPHRASE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_DISABLE_HOSTNAME_VERIFICATION; +import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_ENABLE_AUTOCONFIGURE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_KEYSTORE_FILE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_KEYSTORE_PASSPHRASE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_MASTER; +import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_NAMESPACE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_TLS_VERSIONS; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_TRUSTSTORE_FILE; import static org.apache.karaf.cellar.kubernetes.ConfigKey.KUBERNETES_TRUSTSTORE_PASSPHRASE; @@ -131,8 +134,10 @@ public void updated(String pid, Dictionary properties) throws ConfigurationExcep kubernetesMaster = "http://" + kubernetesHost + ":" + kubernetesPort; } + kubernetesDiscoveryService.setKubernetesAutoconfigure(KUBERNETES_ENABLE_AUTOCONFIGURE.getValue(properties)); kubernetesDiscoveryService.setKubernetesMaster(kubernetesMaster); kubernetesDiscoveryService.setKubernetesApiVersion(KUBERNETES_API_VERSION.getValue(properties)); + kubernetesDiscoveryService.setKubernetesNamespace(KUBERNETES_NAMESPACE.getValue(properties)); kubernetesDiscoveryService.setKubernetesTrustCertificates(KUBERNETES_TRUST_CERTIFICATES.getValue(properties)); kubernetesDiscoveryService.setKubernetesDisableHostnameVerification(KUBERNETES_DISABLE_HOSTNAME_VERIFICATION.getValue(properties)); kubernetesDiscoveryService.setKubernetesCertsCaFile(KUBERNETES_CERTS_CA_FILE.getValue(properties)); diff --git a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java index 01899c30..0d03ec26 100644 --- a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java +++ b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceFactoryTest.java @@ -150,6 +150,7 @@ public void verifyUpdateKubernetesConfig() throws Exception { properties.put(KUBERNETES_KEYSTORE_PASSPHRASE.propertyName, EXPECTED_KUBERNETES_KEYSTORE_PASSPHRASE); update(); assertEquals(EXPECTED_KUBERNETES_API_VERSION, registeredService.getKubernetesApiVersion()); + assertEquals(EXPECTED_KUBERNETES_API_VERSION, registeredService.getKubernetesApiVersion()); assertEquals(Boolean.parseBoolean(EXPECTED_KUBERNETES_TRUST_CERTIFICATES), registeredService.isKubernetesTrustCertificates()); assertEquals(Boolean.parseBoolean(EXPECTED_KUBERNETES_DISABLE_HOSTNAME_VERIFICATION), registeredService.isKubernetesDisableHostnameVerification()); assertEquals(EXPECTED_KUBERNETES_CERTS_CA_FILE, registeredService.getKubernetesCertsCaFile()); diff --git a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java index e6e792e8..d2313f83 100644 --- a/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java +++ b/kubernetes/src/test/java/org/apache/karaf/cellar/kubernetes/KubernetesDiscoveryServiceTest.java @@ -29,6 +29,7 @@ public class KubernetesDiscoveryServiceTest { static final String EXPECTED_KUBERNETES_MASTER = "http://master/"; static final String EXPECTED_KUBERNETES_API_VERSION = "api version"; + static final String EXPECTED_KUBERNETES_NAMESPACE = "default"; static final String EXPECTED_KUBERNETES_TRUST_CERTIFICATES = "true"; static final String EXPECTED_KUBERNETES_DISABLE_HOSTNAME_VERIFICATION = "true"; static final String EXPECTED_KUBERNETES_CERTS_CA_FILE = "certs ca file"; @@ -84,6 +85,7 @@ public void setup() { public void createConfig() { service.setKubernetesMaster(EXPECTED_KUBERNETES_MASTER); service.setKubernetesApiVersion(EXPECTED_KUBERNETES_API_VERSION); + service.setKubernetesNamespace(EXPECTED_KUBERNETES_NAMESPACE); service.setKubernetesTrustCertificates(EXPECTED_KUBERNETES_TRUST_CERTIFICATES); service.setKubernetesDisableHostnameVerification(EXPECTED_KUBERNETES_DISABLE_HOSTNAME_VERIFICATION); service.setKubernetesCertsCaFile(EXPECTED_KUBERNETES_CERTS_CA_FILE); @@ -109,6 +111,7 @@ public void createConfig() { Config config = service.createConfig(); assertEquals(EXPECTED_KUBERNETES_MASTER, config.getMasterUrl()); assertEquals(EXPECTED_KUBERNETES_API_VERSION, config.getApiVersion()); + assertEquals(EXPECTED_KUBERNETES_NAMESPACE, config.getNamespace()); assertTrue(config.isTrustCerts()); assertTrue(config.isDisableHostnameVerification()); assertEquals(EXPECTED_KUBERNETES_CERTS_CA_FILE, config.getCaCertFile()); diff --git a/manual/src/main/asciidoc/user-guide/cloud.adoc b/manual/src/main/asciidoc/user-guide/cloud.adoc index d484b1de..ab5e7ae0 100644 --- a/manual/src/main/asciidoc/user-guide/cloud.adoc +++ b/manual/src/main/asciidoc/user-guide/cloud.adoc @@ -129,4 +129,89 @@ pod.label.value=cellar ---- In case you change the file, the discovery service will check again for new nodes. If new nodes are found, Hazelcast configuration will be -updated and the instance restarted. \ No newline at end of file +updated and the instance restarted. + +===== Hazelcast configuration for Kubernetes + +Even within Kubernetes, the default `hazelcast.xml` will discover nodes on the network via multicast. If this is not desired, make sure to disable multicast discovery and enable TCP/IP in `hazelcast.xml`. You can also disable the port auto-increment, since each container will have a different IP, and a predictable port is preferable: + +---- + + + 5701 + + + + + .... + + .... + + +---- + +===== Using Kubernetes service accounts + +https://kubernetes.io/docs/tasks/configure-pod-container/configure-service-account/[Kubernetes service accounts] allow the processes running within pods to access the Kubernetes API server. Make sure the chosen service account has the appropriate authorization to enumerate Karaf Cellar pods in the appropriate Kubernetes namespace(s). Minimally, allow `get`, `watch` and `list` for `pod` resources: + +---- +apiVersion: rbac.authorization.k8s.io/v1 +kind: Role / ClusterRole +rules: +- apiGroups: + - "" + resources: + - pods + verbs: + - get + - watch + +---- + +===== Kubernetes client automatic configuration + +The Kubernetes client used by Karaf Cellar supports autoconfiguration as described https://github.com/fabric8io/kubernetes-client#configuring-the-client[here]. This is turned off by default in Karaf Cellar. However, this feature can be turned on by using the setting `kubernetes.autoConfig = true` in the Karaf Cellar Kubernetes discovery service. + +Note: not all the options provided by the Kubernetes client are actually usable within Cellar. + +Here is a sample `etc/org.apache.karaf.cellar.kubernetes-mycellarcluster.cfg`: + +---- +pod.label.key=cellar_cluster +pod.label.value=mycluster +kubernetes.autoConfig=true +---- + +Such a setup is suitable to be dropped in a docker container intended to run in a pod configured as follows: + +---- +apiVersion: v1 +kind: Pod +metadata: + labels: + cellar_cluster: mycellarcluster +... +spec: + automountServiceAccountToken: true +... +---- + +===== Manual Kubernetes service account configuration + +If the automatic configuration does not work, a good starting point to replicate the same function manually would be: + +---- +pod.label.key=cellar_cluster +pod.label.value=mycluster + +kubernetes.master=https://$[env:KUBERNETES_SERVICE_HOST]:$[env:KUBERNETES_SERVICE_PORT_HTTPS] +kubernetes.api.version=v1 + +kubernetes.certs.ca.file=/var/run/secrets/kubernetes.io/serviceaccount/ca.crt +kubernetes.auth.token=$[secret:kube-secrets/kubernetes.io/serviceaccount/token] +kubernetes.namespace=$[secret:kube-secrets/kubernetes.io/serviceaccount/namespace] +---- + +The above relies on Karaf's configuration admin environment and secret interpolation. See https://karaf.apache.org/manual/latest/#_secret_files[Secret files] and https://karaf.apache.org/manual/latest/#_environment_variables[Environment variables] in the Karaf manual. You will have to https://kubernetes.io/docs/reference/access-authn-authz/service-accounts-admin/#service-account-admission-controller[configure kubernetes to mount service account secrets] at `${karaf.etc}/kube-secrets`, or, alternatively, provide a symlink towards `/var/run/secrets`, or change the corresponding setting in `config.properties`: