Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Add cache methods for writing/reading local base image container config #2038

Merged
merged 4 commits into from
Oct 4, 2019
Merged
Show file tree
Hide file tree
Changes from 2 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
38 changes: 38 additions & 0 deletions jib-core/src/main/java/com/google/cloud/tools/jib/cache/Cache.java
Original file line number Diff line number Diff line change
Expand Up @@ -130,6 +130,25 @@ public CachedLayer writeTarLayer(DescriptorDigest diffId, Blob compressedBlob)
return cacheStorageWriter.writeTarLayer(diffId, compressedBlob);
}

/**
* Writes a container configuration to {@code (cache directory)/local/config/(image id)}. An image
* ID is a SHA hash of a container configuration JSON. The value is also shown as IMAGE ID in
* {@code docker images}.
*
* <p>Note: the {@code imageID} to the {@code containerConfiguration} is a one-way relationship;
* there is no guarantee that {@code containerConfiguration}'s SHA will be {@code imageID}, since
* the original container configuration is being rewritten here rather than being moved.
*
* @param imageId the ID of the image to store the container configuration for
* @param containerConfiguration the container configuration
* @throws IOException if an I/O exception occurs
*/
void writeLocalConfig(
DescriptorDigest imageId, ContainerConfigurationTemplate containerConfiguration)
throws IOException {
cacheStorageWriter.writeLocalConfig(imageId, containerConfiguration);
}

/**
* Retrieves the cached manifest and container configuration for an image reference.
*
Expand Down Expand Up @@ -187,4 +206,23 @@ public Optional<CachedLayer> retrieveTarLayer(DescriptorDigest diffId)
throws IOException, CacheCorruptedException {
return cacheStorageReader.retrieveTarLayer(diffId);
}

/**
* Retrieves the {@link ContainerConfigurationTemplate} for the image saved from the given image
* ID. An image ID is a SHA hash of a container configuration JSON. The value is also shown as
* IMAGE ID in {@code docker images}.
*
* <p>Note: the {@code imageID} is only used to find the {@code containerConfiguration}, and is
* not necessarily the actual SHA of {@code containerConfiguration}. There is no guarantee that
* {@code containerConfiguration}'s SHA will be {@code imageID}, since the saved container
* configuration is not a direct copy of the base image's original configuration.
*
* @param imageId the image ID
* @return the {@link ContainerConfigurationTemplate} referenced by the image ID, if found
* @throws IOException if an I/O exception occurs
*/
public Optional<ContainerConfigurationTemplate> retrieveLocalConfig(DescriptorDigest imageId)
throws IOException {
return cacheStorageReader.retrieveLocalConfig(imageId);
}
}
Original file line number Diff line number Diff line change
Expand Up @@ -223,6 +223,26 @@ Optional<CachedLayer> retrieveTarLayer(DescriptorDigest diffId)
}
}

/**
* Retrieves the {@link ContainerConfigurationTemplate} for the image with the given image ID.
*
* @param imageId the image ID
* @return the {@link ContainerConfigurationTemplate} referenced by the image ID, if found
* @throws IOException if an I/O exception occurs
*/
Optional<ContainerConfigurationTemplate> retrieveLocalConfig(DescriptorDigest imageId)
throws IOException {
Path configPath =
cacheStorageFiles.getLocalDirectory().resolve("config").resolve(imageId.getHash());
chanseokoh marked this conversation as resolved.
Show resolved Hide resolved
if (!Files.exists(configPath)) {
return Optional.empty();
}

ContainerConfigurationTemplate config =
JsonTemplateMapper.readJsonFromFile(configPath, ContainerConfigurationTemplate.class);
return Optional.of(config);
}

/**
* Retrieves the layer digest selected by the {@code selector}.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -332,6 +332,21 @@ void writeMetadata(ImageReference imageReference, V21ManifestTemplate manifestTe
}
}

/**
* Writes a container configuration to {@code (cache directory)/local/config/(image id)}.
*
* @param imageId the ID of the image to store the container configuration for
* @param containerConfiguration the container configuration
* @throws IOException if an I/O exception occurs
*/
void writeLocalConfig(
DescriptorDigest imageId, ContainerConfigurationTemplate containerConfiguration)
throws IOException {
Path configDirectory = cacheStorageFiles.getLocalDirectory().resolve("config");
Files.createDirectories(configDirectory);
writeMetadata(containerConfiguration, configDirectory.resolve(imageId.getHash()));
}

/**
* Writes a compressed {@code layerBlob} to the {@code layerDirectory}.
*
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -248,6 +248,35 @@ public void testRetrieveTarLayer() throws IOException, CacheCorruptedException {
}
}

@Test
public void testRetrieveLocalConfig() throws IOException, URISyntaxException, DigestException {
Path cacheDirectory = temporaryFolder.newFolder().toPath();
Path configDirectory = cacheDirectory.resolve("local").resolve("config");
Files.createDirectories(configDirectory);
Files.copy(
Paths.get(Resources.getResource("core/json/containerconfig.json").toURI()),
configDirectory.resolve(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"));

CacheStorageFiles cacheStorageFiles = new CacheStorageFiles(cacheDirectory);
CacheStorageReader cacheStorageReader = new CacheStorageReader(cacheStorageFiles);

ContainerConfigurationTemplate configurationTemplate =
cacheStorageReader
.retrieveLocalConfig(
DescriptorDigest.fromHash(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"))
.get();
Assert.assertEquals("wasm", configurationTemplate.getArchitecture());
Assert.assertEquals("js", configurationTemplate.getOs());

Optional<ContainerConfigurationTemplate> missingConfigurationTemplate =
cacheStorageReader.retrieveLocalConfig(
DescriptorDigest.fromHash(
"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"));
Assert.assertFalse(missingConfigurationTemplate.isPresent());
}

@Test
public void testSelect_invalidLayerDigest() throws IOException {
CacheStorageFiles cacheStorageFiles =
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import java.nio.file.Files;
import java.nio.file.Path;
import java.nio.file.Paths;
import java.security.DigestException;
import java.util.zip.GZIPInputStream;
import java.util.zip.GZIPOutputStream;
import org.junit.Assert;
Expand Down Expand Up @@ -185,6 +186,32 @@ public void testWriteMetadata_v22()
Assert.assertEquals("wasm", savedContainerConfig.getArchitecture());
}

@Test
public void testWriteLocalConfig() throws IOException, URISyntaxException, DigestException {
Path containerConfigurationJsonFile =
Paths.get(
getClass().getClassLoader().getResource("core/json/containerconfig.json").toURI());
ContainerConfigurationTemplate containerConfigurationTemplate =
JsonTemplateMapper.readJsonFromFile(
containerConfigurationJsonFile, ContainerConfigurationTemplate.class);

new CacheStorageWriter(cacheStorageFiles)
.writeLocalConfig(
DescriptorDigest.fromHash(
"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"),
containerConfigurationTemplate);

Path savedConfigPath =
cacheStorageFiles
.getLocalDirectory()
.resolve("config")
.resolve("aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa");
Assert.assertTrue(Files.exists(savedConfigPath));
ContainerConfigurationTemplate savedContainerConfig =
JsonTemplateMapper.readJsonFromFile(savedConfigPath, ContainerConfigurationTemplate.class);
Assert.assertEquals("wasm", savedContainerConfig.getArchitecture());
}

private void verifyCachedLayer(CachedLayer cachedLayer, Blob uncompressedLayerBlob)
throws IOException {
BlobDescriptor layerBlobDescriptor = getDigest(compress(uncompressedLayerBlob));
Expand Down