Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Implement KubernetesActuatorTemplate #478

Closed
wants to merge 2 commits into from
Closed
Show file tree
Hide file tree
Changes from 1 commit
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
17 changes: 17 additions & 0 deletions pom.xml
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
<java-semver.version>0.9.0</java-semver.version>
<maven.compiler.plugin.version>3.8.0</maven.compiler.plugin.version>
<powermock.version>2.0.2</powermock.version>
<okhttp.version>3.14.9</okhttp.version>
</properties>


Expand All @@ -40,6 +41,10 @@
<groupId>org.springframework.boot</groupId>
<artifactId>spring-boot-autoconfigure</artifactId>
</dependency>
<dependency>
<groupId>org.springframework</groupId>
<artifactId>spring-web</artifactId>
</dependency>
<dependency>
<groupId>io.fabric8</groupId>
<artifactId>kubernetes-client</artifactId>
Expand Down Expand Up @@ -78,6 +83,18 @@
<version>2.23.4</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>okhttp</artifactId>
<version>${okhttp.version}</version>
<scope>test</scope>
</dependency>
<dependency>
<groupId>com.squareup.okhttp3</groupId>
<artifactId>mockwebserver</artifactId>
<version>${okhttp.version}</version>
<scope>test</scope>
</dependency>
</dependencies>

<profiles>
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2015-2021 the original author or authors.
* Copyright 2015-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -97,9 +97,16 @@ public Container create(ContainerConfiguration containerConfiguration) {

List<String> appArgs = new ArrayList<>();

Map<String, String> appAdminCredentials = new HashMap<>();
properties.getAppAdmin().addCredentialsToAppEnvironmentAsProperties(appAdminCredentials);

switch (entryPointStyle) {
case exec:
appArgs = createCommandArgs(request);
List<String> finalAppArgs = appArgs;
appAdminCredentials.forEach((k, v) -> finalAppArgs.add(String.format("--%s=%s", k,v)));
onobc marked this conversation as resolved.
Show resolved Hide resolved


break;
case boot:
if (envVarsMap.containsKey(SPRING_APPLICATION_JSON)) {
Expand All @@ -121,6 +128,8 @@ public Container create(ContainerConfiguration containerConfiguration) {
for (String key : request.getDefinition().getProperties().keySet()) {
String envVar = key.replace('.', '_').toUpperCase();
envVarsMap.put(envVar, request.getDefinition().getProperties().get(key));
envVarsMap.putAll(appAdminCredentials);

}
// Push all the command line arguments as environment properties
// The task app name(in case of Composed Task), platform_name and executionId are expected to be updated.
Expand Down
Original file line number Diff line number Diff line change
@@ -0,0 +1,41 @@
/*
* Copyright 2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
* You may obtain a copy of the License at
*
* https://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package org.springframework.cloud.deployer.spi.kubernetes;

import org.springframework.cloud.deployer.spi.app.AbstractActuatorTemplate;
import org.springframework.cloud.deployer.spi.app.AppAdmin;
import org.springframework.cloud.deployer.spi.app.AppDeployer;
import org.springframework.cloud.deployer.spi.app.AppInstanceStatus;
import org.springframework.web.client.RestTemplate;

/**
* @author David Turanski
*/

public class KubernetesActuatorTemplate extends AbstractActuatorTemplate {

public KubernetesActuatorTemplate(RestTemplate restTemplate, AppDeployer appDeployer,
AppAdmin appAdmin) {
super(restTemplate, appDeployer, appAdmin);
}

protected String actuatorUrlForInstance(AppInstanceStatus appInstanceStatus) {
return String.format("http://%s:%d/%s", appInstanceStatus.getAttributes().get("pod.ip"),
Integer.valueOf(appInstanceStatus.getAttributes().get("actuator.port")),
appInstanceStatus.getAttributes().get("actuator.path"));
}
}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2015-2020 the original author or authors.
* Copyright 2015-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2015-2019 the original author or authors.
* Copyright 2015-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand All @@ -16,11 +16,14 @@

package org.springframework.cloud.deployer.spi.kubernetes;

import java.util.HashMap;
import java.nio.file.Paths;
import java.util.List;
import java.util.Map;
import java.util.concurrent.ConcurrentHashMap;

import io.fabric8.kubernetes.api.model.Container;
import io.fabric8.kubernetes.api.model.ContainerStatus;
import io.fabric8.kubernetes.api.model.IntOrString;
import io.fabric8.kubernetes.api.model.Pod;
import io.fabric8.kubernetes.api.model.Service;
import io.fabric8.kubernetes.api.model.ServicePort;
Expand All @@ -41,10 +44,15 @@
public class KubernetesAppInstanceStatus implements AppInstanceStatus {

private static Log logger = LogFactory.getLog(KubernetesAppInstanceStatus.class);

private final Pod pod;
private Service service;
private KubernetesDeployerProperties properties;

private final Service service;

private final KubernetesDeployerProperties properties;

private ContainerStatus containerStatus;

private RunningPhaseDeploymentStateResolver runningPhaseDeploymentStateResolver;

@Deprecated
Expand All @@ -62,7 +70,8 @@ public KubernetesAppInstanceStatus(Pod pod, Service service, KubernetesDeployerP
this.runningPhaseDeploymentStateResolver = new DefaultRunningPhaseDeploymentStateResolver(properties);
}

public KubernetesAppInstanceStatus(Pod pod, Service service, KubernetesDeployerProperties properties, ContainerStatus containerStatus) {
public KubernetesAppInstanceStatus(Pod pod, Service service, KubernetesDeployerProperties properties,
ContainerStatus containerStatus) {
this.pod = pod;
this.service = service;
this.properties = properties;
Expand All @@ -73,10 +82,11 @@ public KubernetesAppInstanceStatus(Pod pod, Service service, KubernetesDeployerP
/**
* Override the default {@link RunningPhaseDeploymentStateResolver} implementation.
*
* @param runningPhaseDeploymentStateResolver the {@link RunningPhaseDeploymentStateResolver} to use
* @param runningPhaseDeploymentStateResolver the
* {@link RunningPhaseDeploymentStateResolver} to use
*/
public void setRunningPhaseDeploymentStateResolver(
RunningPhaseDeploymentStateResolver runningPhaseDeploymentStateResolver) {
RunningPhaseDeploymentStateResolver runningPhaseDeploymentStateResolver) {
this.runningPhaseDeploymentStateResolver = runningPhaseDeploymentStateResolver;
}

Expand Down Expand Up @@ -121,28 +131,31 @@ private DeploymentState mapState() {

@Override
public Map<String, String> getAttributes() {
Map<String, String> result = new HashMap<>();

ConcurrentHashMap<String, String> result = new ConcurrentHashMap<>();

if (pod != null) {
result.put("pod.name", pod.getMetadata().getName());
result.put("pod.startTime", pod.getStatus().getStartTime());
result.put("pod.ip", pod.getStatus().getPodIP());
result.put("host.ip", pod.getStatus().getHostIP());
result.put("phase", pod.getStatus().getPhase());
result.put("pod.ip", nullSafe(pod.getStatus().getPodIP()));
result.put("actuator.path", determineActuatorPathFromLivenessProbe(pod));
result.put("actuator.port", determineActuatorPortFromLivenessProbe(pod, result.get("actuator.path")));
result.put("host.ip", nullSafe(pod.getStatus().getHostIP()));
result.put("phase", nullSafe(pod.getStatus().getPhase()));
result.put(AbstractKubernetesDeployer.SPRING_APP_KEY.replace('-', '.'),
pod.getMetadata().getLabels().get(AbstractKubernetesDeployer.SPRING_APP_KEY));
pod.getMetadata().getLabels().get(AbstractKubernetesDeployer.SPRING_APP_KEY));
result.put(AbstractKubernetesDeployer.SPRING_DEPLOYMENT_KEY.replace('-', '.'),
pod.getMetadata().getLabels().get(AbstractKubernetesDeployer.SPRING_DEPLOYMENT_KEY));
pod.getMetadata().getLabels().get(AbstractKubernetesDeployer.SPRING_DEPLOYMENT_KEY));
result.put("guid", pod.getMetadata().getUid());
}
if (service != null) {
result.put("service.name", service.getMetadata().getName());
if ("LoadBalancer".equals(service.getSpec().getType())) {
if (service.getStatus() != null && service.getStatus().getLoadBalancer() != null
&& service.getStatus().getLoadBalancer().getIngress() != null && !service.getStatus()
.getLoadBalancer().getIngress().isEmpty()) {
&& service.getStatus().getLoadBalancer().getIngress() != null && !service.getStatus()
.getLoadBalancer().getIngress().isEmpty()) {
String externalIp = service.getStatus().getLoadBalancer().getIngress().get(0).getIp();
if(externalIp == null) {
if (externalIp == null) {
externalIp = service.getStatus().getLoadBalancer().getIngress().get(0).getHostname();
}
result.put("service.external.ip", externalIp);
Expand All @@ -163,21 +176,41 @@ public Map<String, String> getAttributes() {
result.put("container.restartCount", "" + containerStatus.getRestartCount());
if (containerStatus.getLastState() != null && containerStatus.getLastState().getTerminated() != null) {
result.put("container.lastState.terminated.exitCode",
"" + containerStatus.getLastState().getTerminated().getExitCode());
"" + containerStatus.getLastState().getTerminated().getExitCode());
result.put("container.lastState.terminated.reason",
containerStatus.getLastState().getTerminated().getReason());
containerStatus.getLastState().getTerminated().getReason());
}
if (containerStatus.getState() != null && containerStatus.getState().getTerminated() != null) {
result.put("container.state.terminated.exitCode",
"" + containerStatus.getState().getTerminated().getExitCode());
"" + containerStatus.getState().getTerminated().getExitCode());
result.put("container.state.terminated.reason", containerStatus.getState().getTerminated().getReason());
}
}
return result;
}
}



private String nullSafe(String value) {
return value == null ? "" : value;
}

private String determineActuatorPathFromLivenessProbe(Pod pod) {
return pod.getSpec().getContainers().stream()
.filter((Container container) -> container.getLivenessProbe() != null &&
container.getLivenessProbe().getHttpGet() != null)
.findFirst()
.map(container ->
Paths.get(container.getLivenessProbe().getHttpGet().getPath()).getParent().toString())
.orElse("/actuator");
}

private String determineActuatorPortFromLivenessProbe(Pod pod, String path) {
IntOrString intOrString = pod.getSpec().getContainers().stream()
.filter((Container container) -> container.getLivenessProbe() != null &&
container.getLivenessProbe().getHttpGet() != null &&
container.getLivenessProbe().getHttpGet().getPath().equals(path))
.findFirst()
.map(container -> container.getLivenessProbe().getHttpGet().getPort())
.orElse(new IntOrString(8080));
return intOrString.getIntVal() != null ? String.valueOf(intOrString.getIntVal()) : intOrString.getStrVal();
dturanski marked this conversation as resolved.
Show resolved Hide resolved
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -22,11 +22,13 @@
import org.springframework.boot.autoconfigure.AutoConfigureOrder;
import org.springframework.boot.autoconfigure.condition.ConditionalOnMissingBean;
import org.springframework.boot.context.properties.EnableConfigurationProperties;
import org.springframework.cloud.deployer.spi.app.ActuatorOperations;
import org.springframework.cloud.deployer.spi.app.AppDeployer;
import org.springframework.cloud.deployer.spi.task.TaskLauncher;
import org.springframework.context.annotation.Bean;
import org.springframework.context.annotation.Configuration;
import org.springframework.core.Ordered;
import org.springframework.web.client.RestTemplate;

/**
* Spring Bean configuration for the {@link KubernetesAppDeployer}.
Expand Down Expand Up @@ -72,4 +74,18 @@ public ContainerFactory containerFactory() {
return new DefaultContainerFactory(deployerProperties);
}

@Bean
onobc marked this conversation as resolved.
Show resolved Hide resolved
@ConditionalOnMissingBean(ActuatorOperations.class)
ActuatorOperations actuatorOperations(RestTemplate actuatorRestTemplate, AppDeployer appDeployer,
KubernetesDeployerProperties properties) {
return new KubernetesActuatorTemplate(actuatorRestTemplate, appDeployer, properties.getAppAdmin());
}

@Bean
@ConditionalOnMissingBean
RestTemplate actuatorRestTemplate() {
//TODO: Configure security
return new RestTemplate();
}

}
Original file line number Diff line number Diff line change
@@ -1,5 +1,5 @@
/*
* Copyright 2015-2021 the original author or authors.
* Copyright 2015-2022 the original author or authors.
*
* Licensed under the Apache License, Version 2.0 (the "License");
* you may not use this file except in compliance with the License.
Expand Down Expand Up @@ -28,6 +28,7 @@

import org.springframework.boot.context.properties.ConfigurationProperties;
import org.springframework.boot.context.properties.NestedConfigurationProperty;
import org.springframework.cloud.deployer.spi.app.AppAdmin;

/**
* @author Florian Rosenberg
Expand Down Expand Up @@ -951,6 +952,8 @@ public void setTaskServiceAccountName(String taskServiceAccountName) {
*/
private String deploymentLabels;

private AppAdmin appAdmin = new AppAdmin();

public String getNamespace() {
return namespace;
}
Expand Down Expand Up @@ -1623,4 +1626,12 @@ public String getDeploymentLabels() {
public void setDeploymentLabels(String deploymentLabels) {
this.deploymentLabels = deploymentLabels;
}

public AppAdmin getAppAdmin() {
return appAdmin;
}

public void setAppAdmin(AppAdmin appAdmin) {
this.appAdmin = appAdmin;
}
}
Loading