Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

fix: allow pushing images with different arch/os to docker daemon #4268

Merged
merged 45 commits into from
Jun 25, 2024
Merged
Show file tree
Hide file tree
Changes from 4 commits
Commits
Show all changes
45 commits
Select commit Hold shift + click to select a range
982897e
fix: allow pushing images with different arch/os to docker daemon
mpeddada1 Jun 6, 2024
ffc4df7
fix formatting
mpeddada1 Jun 6, 2024
f2d886b
stubbing
mpeddada1 Jun 6, 2024
9ff90bd
add test for arm64 image
mpeddada1 Jun 7, 2024
62ef484
choose more detailed image name
mpeddada1 Jun 10, 2024
49e84f1
temporarily comment out arm64 test
mpeddada1 Jun 10, 2024
4b62074
fix formatting
mpeddada1 Jun 10, 2024
6a8d809
use different base image
mpeddada1 Jun 10, 2024
ba4cfdf
try simple base image to verify start up
mpeddada1 Jun 10, 2024
8b53402
debug
mpeddada1 Jun 11, 2024
bf39817
undo debugging
mpeddada1 Jun 11, 2024
d3ee6b3
fix arm64 test
mpeddada1 Jun 11, 2024
d98b504
debug macos
mpeddada1 Jun 11, 2024
02cc351
undo workaround
mpeddada1 Jun 11, 2024
74b6c04
revert name change
mpeddada1 Jun 11, 2024
25d6e94
rename test and revert image change
mpeddada1 Jun 11, 2024
2144d08
empty commit
mpeddada1 Jun 11, 2024
2c4afc6
rename to testBasic_toDockerDaemon_arm64
mpeddada1 Jun 11, 2024
d4829f6
debug CI behavior with test names
mpeddada1 Jun 11, 2024
25e12b6
verify with diff arch name
mpeddada1 Jun 11, 2024
3c8a6f0
rename to arm64BaseImage
mpeddada1 Jun 11, 2024
fca0b68
add logic to clean up containers after tests
mpeddada1 Jun 13, 2024
e7a51d8
verify with original name after test resource clean up
mpeddada1 Jun 14, 2024
d92123d
understand order of execution on CI
mpeddada1 Jun 14, 2024
0a6862d
change back to original name
mpeddada1 Jun 14, 2024
8ca43ca
verify image creation
mpeddada1 Jun 14, 2024
9552133
skip distroless_ociManifest test
mpeddada1 Jun 14, 2024
61d528d
add Ignore
mpeddada1 Jun 14, 2024
b762590
verify with lightweight base image
mpeddada1 Jun 14, 2024
1668da6
use separate registry for ociManifest test
mpeddada1 Jun 14, 2024
87d9113
revert to previous image ref
mpeddada1 Jun 14, 2024
908a72b
try different base image
mpeddada1 Jun 14, 2024
693f09c
debug
mpeddada1 Jun 15, 2024
ec31463
docker daemon
mpeddada1 Jun 15, 2024
c32a132
add logging for tear down and move jib build into one test
mpeddada1 Jun 18, 2024
a9f4415
debugging
mpeddada1 Jun 18, 2024
72ab27c
switch order
mpeddada1 Jun 19, 2024
b1c242f
sleep before next build
mpeddada1 Jun 19, 2024
772c883
check with timeout of 120
mpeddada1 Jun 19, 2024
9d51b1f
fromScratch
mpeddada1 Jun 20, 2024
74c25a2
enable all tests
mpeddada1 Jun 20, 2024
3adfb49
add header
mpeddada1 Jun 20, 2024
ebc6c14
clean up tests; fix logging
mpeddada1 Jun 21, 2024
841f207
undo debugging logs
mpeddada1 Jun 21, 2024
71772a6
remove debugging comments
mpeddada1 Jun 24, 2024
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
Original file line number Diff line number Diff line change
Expand Up @@ -17,7 +17,6 @@
package com.google.cloud.tools.jib.api;

import static com.google.common.truth.Truth.assertThat;
import static org.junit.Assert.assertThrows;

import com.google.cloud.tools.jib.Command;
import com.google.cloud.tools.jib.api.buildplan.Platform;
Expand Down Expand Up @@ -321,6 +320,39 @@ public void testBasic_jibImageToDockerDaemon()
Assert.assertEquals("Hello World\n", output);
}

@Test
public void testBasic_jibImageToDockerDaemon_arm64()
throws IOException, InterruptedException, InvalidImageReferenceException, ExecutionException,
RegistryException, CacheDirectoryCreationException {
Jib.from(
RegistryImage.named(
"busybox@sha256:eb427d855f82782c110b48b9a398556c629ce4951ae252c6f6751a136e194668"))
.containerize(
Containerizer.to(
DockerDaemonImage.named(dockerHost + ":5000/docker-daemon-mismatched-arch")));

String os =
new Command(
"docker",
"inspect",
dockerHost + ":5000/docker-daemon-mismatched-arch",
"--format",
"{{.Os}}")
.run()
.replace("\n", "");
String architecture =
new Command(
"docker",
"inspect",
dockerHost + ":5000/docker-daemon-mismatched-arch",
"--format",
"{{.Architecture}}")
.run()
.replace("\n", "");
assertThat(os).isEqualTo("linux");
assertThat(architecture).isEqualTo("arm64");
}

@Test
public void testBasicMultiPlatform_toDockerDaemon()
throws IOException, InterruptedException, ExecutionException, RegistryException,
Expand All @@ -343,27 +375,37 @@ public void testBasicMultiPlatform_toDockerDaemon()
}

@Test
public void testBasicMultiPlatform_toDockerDaemon_noMatchingImage() {
ExecutionException exception =
assertThrows(
ExecutionException.class,
() ->
Jib.from(
RegistryImage.named(
"busybox@sha256:4f47c01fa91355af2865ac10fef5bf6ec9c7f42ad2321377c21e844427972977"))
.setPlatforms(
ImmutableSet.of(
new Platform("s390x", "linux"), new Platform("arm", "linux")))
.setEntrypoint("echo", "Hello World")
.containerize(
Containerizer.to(
DockerDaemonImage.named(
dockerHost + ":5000/docker-daemon-multi-platform"))
.setAllowInsecureRegistries(true)));
assertThat(exception)
.hasCauseThat()
.hasMessageThat()
.startsWith("The configured platforms don't match the Docker Engine's OS and architecture");
public void testBasicMultiPlatform_toDockerDaemon_pickFirstPlatformWhenNoMatchingImage()
mpeddada1 marked this conversation as resolved.
Show resolved Hide resolved
throws IOException, InterruptedException, InvalidImageReferenceException,
CacheDirectoryCreationException, ExecutionException, RegistryException {
Jib.from(
RegistryImage.named(
"busybox@sha256:4f47c01fa91355af2865ac10fef5bf6ec9c7f42ad2321377c21e844427972977"))
.setPlatforms(ImmutableSet.of(new Platform("s390x", "linux"), new Platform("arm", "linux")))
.containerize(
Containerizer.to(
DockerDaemonImage.named(dockerHost + ":5000/docker-daemon-multi-platform"))
.setAllowInsecureRegistries(true));
String os =
new Command(
"docker",
"inspect",
dockerHost + ":5000/docker-daemon-multi-platform",
"--format",
"{{.Os}}")
.run()
.replace("\n", "");
String architecture =
new Command(
"docker",
"inspect",
dockerHost + ":5000/docker-daemon-multi-platform",
"--format",
"{{.Architecture}}")
.run()
.replace("\n", "");
assertThat(os).isEqualTo("linux");
assertThat(architecture).isEqualTo("s390x");
}

@Test
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -624,14 +624,9 @@ private void loadDocker(
DockerInfoDetails dockerInfoDetails = dockerClient.info();
String osType = dockerInfoDetails.getOsType();
String architecture = normalizeArchitecture(dockerInfoDetails.getArchitecture());
Optional<Image> builtImage = fetchBuiltImageForLocalBuild(osType, architecture);
Preconditions.checkState(
builtImage.isPresent(),
String.format(
"The configured platforms don't match the Docker Engine's OS and architecture (%s/%s)",
osType, architecture));
Image builtImage = fetchBuiltImageForLocalBuild(osType, architecture);
return new LoadDockerStep(
buildContext, progressDispatcherFactory, dockerClient, builtImage.get())
buildContext, progressDispatcherFactory, dockerClient, builtImage)
.call();
});
}
Expand Down Expand Up @@ -669,21 +664,22 @@ String normalizeArchitecture(String architecture) {
}

@VisibleForTesting
Optional<Image> fetchBuiltImageForLocalBuild(String osType, String architecture)
Image fetchBuiltImageForLocalBuild(String osType, String architecture)
throws InterruptedException, ExecutionException {
if (results.baseImagesAndBuiltImages.get().size() > 1) {
LOGGER.warning(
String.format(
"Detected multi-platform configuration, only building the one that matches the local Docker Engine's os and architecture (%s/%s)",
"Detected multi-platform configuration, only building the one that matches the local Docker Engine's os and architecture (%s/%s) or "
+ "the first platform specified",
osType, architecture));
}
for (Map.Entry<Image, Future<Image>> imageEntry :
results.baseImagesAndBuiltImages.get().entrySet()) {
Image image = imageEntry.getValue().get();
if (image.getArchitecture().equals(architecture) && image.getOs().equals(osType)) {
return Optional.of(image);
for (Map.Entry<Image, Future<Image>> imageEntry :
results.baseImagesAndBuiltImages.get().entrySet()) {
Image image = imageEntry.getValue().get();
if (image.getArchitecture().equals(architecture) && image.getOs().equals(osType)) {
return image;
}
}
}
return Optional.empty();
return results.baseImagesAndBuiltImages.get().values().iterator().next().get();
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -203,14 +203,14 @@ public void testFetchBuildImageForLocalBuild_matchingOsAndArch()
Futures.immediateFuture(builtAmd64AndWindowsImage))));
stepsRunner.buildImages(progressDispatcherFactory);

Optional<Image> expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("windows", "amd64");
Image expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("windows", "amd64");

assertThat(expectedImage.get().getOs()).isEqualTo("windows");
assertThat(expectedImage.get().getArchitecture()).isEqualTo("amd64");
assertThat(expectedImage.getOs()).isEqualTo("windows");
assertThat(expectedImage.getArchitecture()).isEqualTo("amd64");
}

@Test
public void testFetchBuildImageForLocalBuild_differentOs()
public void testFetchBuildImageForLocalBuild_differentOs_buildImageForFirstPlatform()
throws ExecutionException, InterruptedException {
when(builtArm64AndLinuxImage.getArchitecture()).thenReturn("arm64");
when(builtArm64AndLinuxImage.getOs()).thenReturn("linux");
Expand All @@ -225,13 +225,14 @@ public void testFetchBuildImageForLocalBuild_differentOs()
Futures.immediateFuture(builtAmd64AndWindowsImage))));
stepsRunner.buildImages(progressDispatcherFactory);

Optional<Image> expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("os", "arm64");
Image expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("os", "arm64");

assertThat(expectedImage.isPresent()).isFalse();
assertThat(expectedImage.getOs()).isEqualTo("linux");
assertThat(expectedImage.getArchitecture()).isEqualTo("arm64");
}

@Test
public void testFetchBuildImageForLocalBuild_differentArch()
public void testFetchBuildImageForLocalBuild_differentArch_buildImageForFirstPlatform()
throws ExecutionException, InterruptedException {
when(builtArm64AndLinuxImage.getArchitecture()).thenReturn("arm64");
when(builtAmd64AndWindowsImage.getArchitecture()).thenReturn("amd64");
Expand All @@ -245,9 +246,26 @@ public void testFetchBuildImageForLocalBuild_differentArch()
Futures.immediateFuture(builtAmd64AndWindowsImage))));
stepsRunner.buildImages(progressDispatcherFactory);

Optional<Image> expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("linux", "arch");
Image expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("linux", "arch");

assertThat(expectedImage.isPresent()).isFalse();
assertThat(expectedImage.getArchitecture()).isEqualTo("arm64");
}

@Test
public void testFetchBuildImageForLocalBuild_singleImage_imagePlatformDifferentFromDockerEnv()
throws ExecutionException, InterruptedException {
when(builtArm64AndLinuxImage.getArchitecture()).thenReturn("arm64");
when(builtArm64AndLinuxImage.getOs()).thenReturn("linux");
when(executorService.submit(Mockito.any(Callable.class)))
.thenReturn(
Futures.immediateFuture(
ImmutableMap.of(baseImage1, Futures.immediateFuture(builtArm64AndLinuxImage))));
stepsRunner.buildImages(progressDispatcherFactory);

Image expectedImage = stepsRunner.fetchBuiltImageForLocalBuild("linux", "amd64");

assertThat(expectedImage.getOs()).isEqualTo("linux");
assertThat(expectedImage.getArchitecture()).isEqualTo("arm64");
}

@Test
Expand Down
Loading