diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml
index 532921b1ff2..0008e902ca3 100644
--- a/.github/workflows/build-container-image.yml
+++ b/.github/workflows/build-container-image.yml
@@ -4,6 +4,9 @@ on:
       platforms:
         required: true
         type: string
+      cache:
+        type: boolean
+        default: true
       use_native_arm64_builder:
         type: boolean
       push_to_images:
@@ -92,5 +95,5 @@ jobs:
           push: ${{ inputs.push_to_images != '' }}
           tags: ${{ steps.meta.outputs.tags }}
           labels: ${{ steps.meta.outputs.labels }}
-          cache-from: type=gha
-          cache-to: type=gha,mode=max
+          cache-from: ${{ inputs.cache && 'type=gha' }}
+          cache-to: ${{ inputs.cache && 'type=gha,mode=max' }}
diff --git a/.github/workflows/build-nightly.yml b/.github/workflows/build-nightly.yml
index 5feb2ade333..aa1f916afe3 100644
--- a/.github/workflows/build-nightly.yml
+++ b/.github/workflows/build-nightly.yml
@@ -26,6 +26,7 @@ jobs:
     with:
       platforms: linux/amd64,linux/arm64
       use_native_arm64_builder: true
+      cache: false
       push_to_images: |
         tootsuite/mastodon
         ghcr.io/mastodon/mastodon
diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml
index f739a69d9ad..98b9dfc0dfb 100644
--- a/.github/workflows/build-releases.yml
+++ b/.github/workflows/build-releases.yml
@@ -17,6 +17,8 @@ jobs:
       push_to_images: |
         tootsuite/mastodon
         ghcr.io/mastodon/mastodon
+      # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages
+      cache: false
       # Only tag with latest when ran against the latest stable branch
       # This needs to be updated after each minor version release
       flavor: |
diff --git a/Dockerfile b/Dockerfile
index 3fe4a62bdf9..f73bdcf7861 100644
--- a/Dockerfile
+++ b/Dockerfile
@@ -17,6 +17,7 @@ COPY Gemfile* package.json yarn.lock /opt/mastodon/
 
 # hadolint ignore=DL3008
 RUN apt-get update && \
+    apt-get -yq dist-upgrade && \
     apt-get install -y --no-install-recommends build-essential \
         git \
         libicu-dev \