Combining Jenkins With Earthbuild and a Shared Buildkit Daemon
At my day job, we have been using EarthBuild (a fork of Earthly, which has been discontinued) to run the majority of our CI/CD workflows. We are responsible for hundreds of build pipelines running on our shared infrastructure. While this tool has a significant learning curve, it addresses a number of issues that I have seen numerous times as someone who is the "ci/cd expert"
-
Allows "shared functions" so that things like "use artifactory to install apt packages" or "install the company CA certs in the docker image" are easy to do correctly, instead of a shared image that never gets updated or copy and pasting the same 30 lines of code everywhere
-
Much better management of secrets in docker builds
-
The ability to share caches between executions of builds, or even between projects
-
A syntax to make modular Dockerfiles much easier to work with
-
The ability to run end-to-end integration tests entirely within the tool, so the same command can work both locally and in CI
We have learned a lot and written some common code that we believe makes the whole process much smoother.
Jenkins Pipelines in Kubernetes
We run all our Jenkins pipelines in Kubernetes. Instead of provisioning each build pod with tons of CPU and RAM, or building bespoke images for every project, we now use a shared pod spec. That pod spec includes:
-
A Jenkins agent
-
A Docker daemon (running in a sidecar)
-
The Earthbuild CLI
It was easy to wrap this into a Jenkins shared library function. It:
-
Sets up the pod spec
-
Configures Earthbuild to point at our shared BuildKit daemon
-
Sets up credentials and other per-job config
This has saved us a ton of boilerplate. A lot of Jenkinsfiles would bake down to single line earthly commands in each stage.
def call(Closure body) {
pod {
creds {
doCheckout {
sh(script: '''
echo "${ARTIFACTORY_PSW}" | docker login -u "${ARTIFACTORY_USR}" --password-stdin "$BASE_REGISTRY"
''', returnStdout: true)
echo "BuildKit Pod List:\n${env.POD_LIST}"
// Use a hash of the Jenkins job-name to assign the build to a specific buildkit
// instance
script {
def podList = env.POD_LIST.split()
def jobName = env.JOB_NAME ?: "unknown-job"
def hash = jobName.hashCode() & 0x7FFFFFFF // Ensure positive integer
def idx = hash % podList.size()
def selectedPod = podList[idx]
echo "Selected pod: ${selectedPod}"
env.BUILDKIT_POD = selectedPod
}
sh(script: """
earthly account logout
earthly config git "{our.bitbucket.com: {auth: 'ssh', port: 7999, user: 'git', strict_host_key_checking : false}}"
earthly config global.buildkit_host tcp://${env.BUILDKIT_POD}.buildkit.buildkit.svc.cluster.local:8372
earthly config global.tls_enabled false
""", returnStdout: true)
withNewSpan(label: 'earthly-step'){
body()
}
}
}
}
}
// Jenkins sets these variables, so we can use them
// in things like versioning schemes or sonarqube scans
def doCheckout(Closure body) {
def scmVars
scmVars = checkout scm
vars = ["BRANCH_NAME=${env.BRANCH_NAME}"]
if ( env.CHANGE_BRANCH ) {
vars << "CHANGE_BRANCH=${env.CHANGE_BRANCH}"
vars << "CHANGE_TARGET=${env.CHANGE_TARGET}"
vars << "CHANGE_ID=${env.CHANGE_ID}"
}
withEnv([
"GIT_COMMIT=${scmVars.GIT_COMMIT}",
"EARTHLY_BUILD_ARGS=${vars.join(',')}",
]) {
body()
}
}
def pod(Closure body) {
podTemplate(
yamlMergeStrategy: merge(),
yaml: buildTemplate(),
showRawYaml: false
) {
node(POD_LABEL) {
container('netshoot') {
script {
def result = sh(
script: '''
nslookup -type=SRV buildkit.buildkit.svc.cluster.local \
| awk '/service =/ { print $NF }' \
| sed 's/\\.buildkit\\.buildkit\\.svc\\.cluster\\.local\\.$//' \
| sort -u
''',
returnStdout: true
).trim()
// Save to environment variable for later stages
env.POD_LIST = result
}
}
container("docker") {
body()
}
}
}
}
def creds(Closure body) {
sshagent(['bitbucket-ssh-credentials']) {
withCredentials([
usernamePassword(credentialsId: "artifactory", passwordVariable: "ARTIFACTORY_PSW", usernameVariable: "ARTIFACTORY_USR"),
usernamePassword(credentialsId: 'bitbucket', usernameVariable: 'BITBUCKET_USR', passwordVariable: 'BITBUCKET_PSW'),
string(credentialsId: 'sonarqube_token', variable: 'SONAR_TOKEN'),
]) {
secret_names = [
"ARTIFACTORY_USR",
"ARTIFACTORY_PSW",
"BITBUCKET_USR",
"BITBUCKET_PSW",
"SONAR_TOKEN",
]
existing_secrets = env.EARTHLY_SECRETS ? env.EARTHLY_SECRETS.split(",").collect { it.trim() }.toSet() : [] as Set
for (name in secret_names) {
existing_secrets.add(name)
}
earthy_secrets = existing_secrets.join(",")
withEnv([
"EARTHLY_SECRETS=$earthy_secrets"
]){
secretFiles {
body()
}
}
}
}
}
// We mount a kubernetes netrc file in the pods for artifactory credentials
// most of our shared earthfiles use it to install packages
def secretFiles(Closure body) {
withEnv([
'EARTHLY_SECRET_FILES=netrc=/credentials/netrc',
]) {
body()
}
}
def buildTemplate() {
return """
apiVersion: v1
kind: Pod
spec:
containers:
- name: jnlp
securityContext:
runAsUser: 0
- name: docker
image: earthly/earthly:v0.8.15
imagePullPolicy: IfNotPresent
command:
- cat
tty: true
securityContext:
runAsUser: 0
env:
- name: DOCKER_HOST
value: 'tcp://localhost:2376'
- name: DOCKER_TLS_VERIFY
value: '1'
- name: DOCKER_CERT_PATH
value: /certs/client
volumeMounts:
- name: dind-certs
mountPath: /certs
- name: art-saas-creds
mountPath: "/credentials"
- name: dind
image: docker:24.0.5-dind
imagePullPolicy: IfNotPresent
securityContext:
privileged: true
runAsUser: 0
args:
- '--experimental'
resources:
limits:
memory: 4Gi
requests:
cpu: 2000m
memory: 4Gi
volumeMounts:
- name: dind-storage
mountPath: /var/lib/docker
- name: dind-certs
mountPath: /certs
- name: netshoot
image: mktxv-docker-prod-virtual.artifacts.tools.marketaxess.com/nicolaka/netshoot
imagePullPolicy: IfNotPresent
command:
- cat
tty: true
securityContext:
runAsUser: 0
volumes:
- name: dind-storage
emptyDir: {}
- name: dind-certs
emptyDir: {}
- name: art-saas-creds
secret:
secretName: artifactory-saas-netrc
items:
- key: netrc
path: netrc
"""
}
Disk IOPS Issues
The biggest challenge with BuildKit wasn't CPU or memory—it was disk.
BuildKit is extremely disk-intensive, especially when used in parallel across multiple builds. Our Kubernetes cluster runs on AWS, and our first attempt used EBS-backed PVCs to store the BuildKit state.
The result? IOPS starvation. Towards the end of the day when our cluster got extra busy (everyone trying to run get their PR builds in for the 3pm QA deploy), builds started really starting to drag out. This was evidenced by very long "pauses" in the build output, where steps seemed to be doing nothing in particular for long periods. While troubleshooting, AWS EBS volumes have a metric for throughput/IOPS throttle, and we could see this metric was high while we were having issues.
In a past life, the IOPS of EBS volumes was automatically tied to the size of the disk, so I had learned to always make the disks large. However, modern volume types give you more control. To fix this, we provisioned dedicated gp3 volumes ahead of time with the maximum throughput and IOPS supported (16,000 IOPS and 1,000 MB/s), then bound them explicitly to PVCs. BuildKit performance improved dramatically after that. We actually found that this change made much more difference than adding more CPU or RAM to the container, or even scaling to multiple buildkit instances.
Full Terraform Example
We use the following Terraform to provision BuildKit in Kubernetes. This includes:
-
A high-throughput StorageClass
-
Pre-provisioned EBS volumes and bound PVs/PVCs
-
A BuildKit StatefulSet
-
A headless Service for gRPC communication
resource "kubernetes_namespace" "buildkit" {
metadata {
name = "buildkit"
}
}
# This was needed because the docker image hardcodes the buildkit network IP
# range, and it happened to be the same as my eks VPC, so we provided
# a custom copy. We also experimented with changing the garbage collection
# settings this way, but didn't find any that made a big difference
resource "kubernetes_config_map" "docker_wrapper" {
metadata {
name = "docker-wrapper"
namespace = kubernetes_namespace.buildkit.metadata.0.name
}
data = {
"dockerd-wrapper.sh" = "${file(\"${path.module}/earthly/dockerd-wrapper.sh\")}"
}
}
# I have since learned that I could have added the throughput to the storage class
# instead of creating the volumes directly
resource "kubernetes_storage_class" "buildkit" {
metadata {
name = "buildkit"
}
storage_provisioner = "ebs.csi.aws.com"
parameters = {
"csi.storage.k8s.io/fstype" = "ext4"
encrypted = "true"
type = "gp3"
}
reclaim_policy = "Delete"
mount_options = [
"context=\"system_u:object_r:local_t:s0\"",
]
}
resource "aws_ebs_volume" "buildkit" {
count = var.buildkit_replicas
availability_zone = var.az
size = 1000
type = "gp3"
iops = 16000
throughput = 1000
tags = {
Name = "buildkit-volume-${count.index}"
}
}
resource "kubernetes_persistent_volume" "buildkit" {
count = var.buildkit_replicas
metadata {
name = "buildkit-volume-${count.index}"
}
spec {
capacity = {
storage = "1000Gi"
}
access_modes = ["ReadWriteOnce"]
persistent_volume_reclaim_policy = "Retain"
mount_options = [
"context=\"system_u:object_r:local_t:s0\"",
]
node_affinity {
required {
node_selector_term {
match_expressions {
key = "topology.ebs.csi.aws.com/zone"
operator = "In"
values = [var.az]
}
}
}
}
storage_class_name = kubernetes_storage_class.buildkit.metadata.0.name
persistent_volume_source {
csi {
driver = "ebs.csi.aws.com"
fs_type = "ext4"
volume_handle = aws_ebs_volume.buildkit[count.index].id
}
}
}
}
resource "kubernetes_persistent_volume_claim" "buildkit" {
count = var.buildkit_replicas
wait_until_bound = false
metadata {
name = "buildkit-volume-buildkit-${count.index}"
namespace = kubernetes_namespace.buildkit.metadata.0.name
}
spec {
access_modes = ["ReadWriteOnce"]
resources {
requests = {
storage = "1000Gi"
}
}
storage_class_name = kubernetes_storage_class.buildkit.metadata.0.name
volume_name = kubernetes_persistent_volume.buildkit[count.index].metadata.0.name
}
}
resource "kubernetes_stateful_set" "buildkit" {
metadata {
name = "buildkit"
namespace = kubernetes_namespace.buildkit.metadata.0.name
}
spec {
pod_management_policy = "Parallel"
replicas = var.buildkit_replicas
revision_history_limit = 3
service_name = "buildkit"
selector {
match_labels = {
k8s-app = "buildkit"
}
}
template {
metadata {
labels = {
k8s-app = "buildkit"
}
}
spec {
node_selector = {
"karpenter.k8s.aws/instance-family" = "m7i"
"karpenter.sh/capacity-type" = "on-demand"
"topology.kubernetes.io/zone" = var.az
}
# The init container enables QEMU on the nodes so that
# you can do cross-platform builds. Remove this if thats
# not necessary
init_container {
name = "binfmt"
image = "tonistiigi/binfmt:master"
args = ["--install", "all"]
security_context {
privileged = true
}
}
container {
name = "buildkitd"
image = "earthly/buildkitd:v0.8.16-ticktock"
security_context {
privileged = true
run_as_user = 0
allow_privilege_escalation = true
capabilities {
add = [
"SYS_ADMIN",
"SETGID",
"SETUID",
"SYS_PTRACE",
"SYS_CHROOT",
"DAC_OVERRIDE",
"SETPCAP",
"DAC_READ_SEARCH",
"BPF",
"PERFMON",
"SYS_RESOURCE",
"NET_RAW",
"CHOWN",
"NET_ADMIN",
]
}
}
volume_mount {
mount_path = "/tmp/earthly"
name = "buildkit-volume"
}
volume_mount {
mount_path = "/var/earthly/dockerd-wrapper.sh"
name = "dockerd-wrapper"
sub_path = "dockerd-wrapper.sh"
}
resources {
requests = {
cpu = "${var.buildkit_cpu}"
memory = "${var.buildkit_cpu * 4}Gi"
}
limits = {
memory = "${var.buildkit_cpu * 4}Gi"
}
}
env {
name = "HOST_IP"
value_from {
field_ref {
field_path = "status.hostIP"
}
}
}
env {
name = "POD_IP"
value_from {
field_ref {
field_path = "status.podIP"
}
}
}
env {
name = "BUILDKIT_TCP_TRANSPORT_ENABLED"
value = "true"
}
env {
name = "BUILDKIT_TLS_ENABLED"
value = "false"
}
env {
name = "BUILDKIT_MAX_PARALLELISM"
value = "100"
}
env {
name = "CACHE_SIZE_PCT"
value = "75"
}
env {
name = "BUILDKIT_DEBUG"
value = "true"
}
env {
name = "BUILDKIT_PPROF_ENABLED"
value = "true"
}
}
volume {
name = "dockerd-wrapper"
config_map {
name = kubernetes_config_map.docker_wrapper.metadata.0.name
default_mode = "0777"
}
}
}
}
volume_claim_template {
metadata {
name = "buildkit-volume"
namespace = kubernetes_namespace.buildkit.metadata.0.name
}
spec {
access_modes = ["ReadWriteOnce"]
resources {
requests = {
storage = "1000Gi"
}
}
storage_class_name = kubernetes_storage_class.buildkit.metadata.0.name
}
}
}
depends_on = [kubernetes_persistent_volume_claim.buildkit]
}
# Need the headless service so that we can pin specific builds
# to specific instances of buildkit for caching
resource "kubernetes_service" "buildkit" {
metadata {
name = "buildkit"
namespace = kubernetes_namespace.buildkit.metadata.0.name
}
spec {
cluster_ip = "None"
selector = {
k8s-app = "buildkit"
}
session_affinity = "ClientIP"
session_affinity_config {
client_ip {
timeout_seconds = 3000
}
}
port {
name = "grpc"
port = 8372
target_port = 8372
protocol = "TCP"
}
port {
name = "debug"
port = 6060
target_port = 6060
protocol = "TCP"
}
}
}