diff --git a/.github/workflows/build-container-image.yml b/.github/workflows/build-container-image.yml index 29868c72f8a..e100e158214 100644 --- a/.github/workflows/build-container-image.yml +++ b/.github/workflows/build-container-image.yml @@ -21,6 +21,8 @@ on: type: string labels: type: string + file_to_build: + type: string jobs: build-image: @@ -86,6 +88,7 @@ jobs: - uses: docker/build-push-action@v5 with: context: . + file: ${{ inputs.file_to_build }} build-args: | MASTODON_VERSION_PRERELEASE=${{ inputs.version_prerelease }} MASTODON_VERSION_METADATA=${{ inputs.version_metadata }} diff --git a/.github/workflows/build-nightly.yml b/.github/workflows/build-nightly.yml index 1790d5c84fa..7c6f74b4571 100644 --- a/.github/workflows/build-nightly.yml +++ b/.github/workflows/build-nightly.yml @@ -25,6 +25,7 @@ jobs: needs: compute-suffix uses: ./.github/workflows/build-container-image.yml with: + file_to_build: Dockerfile platforms: linux/amd64,linux/arm64 use_native_arm64_builder: true cache: false @@ -41,3 +42,25 @@ jobs: type=raw,value=nightly type=schedule,pattern=${{ needs.compute-suffix.outputs.prerelease }} secrets: inherit + + build-image-streaming: + needs: compute-suffix + uses: ./.github/workflows/build-container-image.yml + with: + file_to_build: streaming/Dockerfile + platforms: linux/amd64,linux/arm64 + use_native_arm64_builder: true + cache: false + push_to_images: | + tootsuite/mastodon-streaming + ghcr.io/mastodon/mastodon-streaming + version_prerelease: ${{ needs.compute-suffix.outputs.prerelease }} + labels: | + org.opencontainers.image.description=Nightly build image used for testing purposes + flavor: | + latest=auto + tags: | + type=raw,value=edge + type=raw,value=nightly + type=schedule,pattern=${{ needs.compute-suffix.outputs.prerelease }} + secrets: inherit diff --git a/.github/workflows/build-push-pr.yml b/.github/workflows/build-push-pr.yml index 1f647e2a141..72baed5121a 100644 --- a/.github/workflows/build-push-pr.yml +++ b/.github/workflows/build-push-pr.yml @@ -29,6 +29,7 @@ jobs: needs: compute-suffix uses: ./.github/workflows/build-container-image.yml with: + file_to_build: Dockerfile platforms: linux/amd64,linux/arm64 use_native_arm64_builder: true push_to_images: | @@ -39,3 +40,19 @@ jobs: tags: | type=ref,event=pr secrets: inherit + + build-image-streaming: + needs: compute-suffix + uses: ./.github/workflows/build-container-image.yml + with: + file_to_build: streaming/Dockerfile + platforms: linux/amd64,linux/arm64 + use_native_arm64_builder: true + push_to_images: | + ghcr.io/mastodon/mastodon-streaming + version_metadata: ${{ needs.compute-suffix.outputs.metadata }} + flavor: | + latest=auto + tags: | + type=ref,event=pr + secrets: inherit diff --git a/.github/workflows/build-releases.yml b/.github/workflows/build-releases.yml index 3b82eef9d89..3f0bef32ac4 100644 --- a/.github/workflows/build-releases.yml +++ b/.github/workflows/build-releases.yml @@ -12,6 +12,7 @@ jobs: build-image: uses: ./.github/workflows/build-container-image.yml with: + file_to_build: Dockerfile platforms: linux/amd64,linux/arm64 use_native_arm64_builder: true push_to_images: | @@ -27,3 +28,24 @@ jobs: type=pep440,pattern={{raw}} type=pep440,pattern=v{{major}}.{{minor}} secrets: inherit + + build-image-streaming: + if: startsWith(github.ref, 'refs/tags/v4.3.') + uses: ./.github/workflows/build-container-image.yml + with: + file_to_build: streaming/Dockerfile + platforms: linux/amd64,linux/arm64 + use_native_arm64_builder: true + push_to_images: | + tootsuite/mastodon-streaming + ghcr.io/mastodon/mastodon-streaming + # Do not use cache when building releases, so apt update is always ran and the release always contain the latest packages + cache: false + # Only tag with latest when ran against the latest stable branch + # This needs to be updated after each minor version release + flavor: | + latest=${{ startsWith(github.ref, 'refs/tags/v4.3.') }} + tags: | + type=pep440,pattern={{raw}} + type=pep440,pattern=v{{major}}.{{minor}} + secrets: inherit diff --git a/.github/workflows/test-image-build.yml b/.github/workflows/test-image-build.yml index 778e341771e..980e071897c 100644 --- a/.github/workflows/test-image-build.yml +++ b/.github/workflows/test-image-build.yml @@ -7,6 +7,7 @@ on: - .github/workflows/build-releases.yml - .github/workflows/test-image-build.yml - Dockerfile + - streaming/Dockerfile permissions: contents: read @@ -18,4 +19,17 @@ jobs: uses: ./.github/workflows/build-container-image.yml with: + file_to_build: Dockerfile platforms: linux/amd64 # Testing only on native platform so it is performant + cache: true + + build-image-streaming: + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-streaming + cancel-in-progress: true + + uses: ./.github/workflows/build-container-image.yml + with: + file_to_build: streaming/Dockerfile + platforms: linux/amd64 # Testing only on native platform so it is performant + cache: true diff --git a/Dockerfile b/Dockerfile index 7e032073b81..623a4d4eab5 100644 --- a/Dockerfile +++ b/Dockerfile @@ -1,20 +1,182 @@ # syntax=docker/dockerfile:1.4 -# This needs to be bookworm-slim because the Ruby image is built on bookworm-slim -ARG NODE_VERSION="20.9-bookworm-slim" -FROM ghcr.io/moritzheiber/ruby-jemalloc:3.2.2-slim as ruby -FROM node:${NODE_VERSION} as build +# Please see https://docs.docker.com/engine/reference/builder for information about +# the extended buildx capabilities used in this file. +# Make sure multiarch TARGETPLATFORM is available for interpolation +# See: https://docs.docker.com/build/building/multi-platform/ +ARG TARGETPLATFORM=${TARGETPLATFORM} +ARG BUILDPLATFORM=${BUILDPLATFORM} -COPY --link --from=ruby /opt/ruby /opt/ruby +# Ruby image to use for base image, change with [--build-arg RUBY_VERSION="3.2.2"] +ARG RUBY_VERSION="3.2.2" +# # Node version to use in base image, change with [--build-arg NODE_MAJOR_VERSION="20"] +ARG NODE_MAJOR_VERSION="20" +# Debian image to use for base image, change with [--build-arg DEBIAN_VERSION="bookworm"] +ARG DEBIAN_VERSION="bookworm" +# Node image to use for base image based on combined variables (ex: 20-bookworm-slim) +FROM docker.io/node:${NODE_MAJOR_VERSION}-${DEBIAN_VERSION}-slim as node +# Ruby image to use for base image based on combined variables (ex: 3.2.2-slim-bookworm) +FROM docker.io/ruby:${RUBY_VERSION}-slim-${DEBIAN_VERSION} as ruby -ENV DEBIAN_FRONTEND="noninteractive" \ - PATH="${PATH}:/opt/ruby/bin" +# Resulting version string is vX.X.X-MASTODON_VERSION_PRERELEASE+MASTODON_VERSION_METADATA +# Example: v4.2.0-nightly.2023.11.09+something +# Overwrite existance of 'alpha.0' in version.rb [--build-arg MASTODON_VERSION_PRERELEASE="nightly.2023.11.09"] +ARG MASTODON_VERSION_PRERELEASE="" +# Append build metadata or fork information to version.rb [--build-arg MASTODON_VERSION_METADATA="something"] +ARG MASTODON_VERSION_METADATA="" -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Allow Ruby on Rails to serve static files +# See: https://docs.joinmastodon.org/admin/config/#rails_serve_static_files +ARG RAILS_SERVE_STATIC_FILES="true" +# Allow to use YJIT compiler +# See: https://github.com/ruby/ruby/blob/master/doc/yjit/yjit.md +ARG RUBY_YJIT_ENABLE="1" +# Timezone used by the Docker container and runtime, change with [--build-arg TZ=Europe/Berlin] +ARG TZ="Etc/UTC" +# Linux UID (user id) for the mastodon user, change with [--build-arg UID=1234] +ARG UID="991" +# Linux GID (group id) for the mastodon user, change with [--build-arg GID=1234] +ARG GID="991" +# Apply Mastodon build options based on options above +ENV \ +# Apply Mastodon version information + MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \ + MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}" \ +# Apply Mastodon static files and YJIT options + RAILS_SERVE_STATIC_FILES=${RAILS_SERVE_STATIC_FILES} \ + RUBY_YJIT_ENABLE=${RUBY_YJIT_ENABLE} \ +# Apply timezone + TZ=${TZ} + +ENV \ +# Configure the IP to bind Mastodon to when serving traffic + BIND="0.0.0.0" \ +# Use production settings for Yarn, Node and related nodejs based tools + NODE_ENV="production" \ +# Use production settings for Ruby on Rails + RAILS_ENV="production" \ +# Add Ruby and Mastodon installation to the PATH + DEBIAN_FRONTEND="noninteractive" \ + PATH="${PATH}:/opt/ruby/bin:/opt/mastodon/bin" \ +# Optimize jemalloc 5.x performance + MALLOC_CONF="narenas:2,background_thread:true,thp:never,dirty_decay_ms:1000,muzzy_decay_ms:0" + +# Set default shell used for running commands +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-c"] + +ARG TARGETPLATFORM + +RUN echo "Target platform is $TARGETPLATFORM" + +RUN \ +# Sets timezone + echo "${TZ}" > /etc/localtime; \ +# Creates mastodon user/group and sets home directory + groupadd -g "${GID}" mastodon; \ + useradd -l -u "${UID}" -g "${GID}" -m -d /opt/mastodon mastodon; \ +# Creates /mastodon symlink to /opt/mastodon + ln -s /opt/mastodon /mastodon; + +# Set /opt/mastodon as working directory WORKDIR /opt/mastodon +# hadolint ignore=DL3008,DL3005 +RUN \ +# Mount Apt cache and lib directories from Docker buildx caches +--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \ +--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \ +# Apt update & upgrade to check for security updates to Debian image + apt-get update; \ + apt-get dist-upgrade -yq; \ +# Install jemalloc, curl and other necessary components + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + ffmpeg \ + file \ + imagemagick \ + libjemalloc2 \ + patchelf \ + procps \ + tini \ + tzdata \ + ; \ +# Patch Ruby to use jemalloc + patchelf --add-needed libjemalloc.so.2 /usr/local/bin/ruby; \ +# Discard patchelf after use + apt-get purge -y \ + patchelf \ + ; + +# Create temporary build layer from base image +FROM ruby as build + +# Copy Node package configuration files into working directory +COPY package.json yarn.lock .yarnrc.yml /opt/mastodon/ +COPY .yarn /opt/mastodon/.yarn + +COPY --from=node /usr/local/bin /usr/local/bin +COPY --from=node /usr/local/lib /usr/local/lib + +ARG TARGETPLATFORM + # hadolint ignore=DL3008 +RUN \ +# Mount Apt cache and lib directories from Docker buildx caches +--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \ +--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \ +# Install build tools and bundler dependencies from APT + apt-get update; \ + apt-get install -y --no-install-recommends \ + g++ \ + gcc \ + git \ + libgdbm-dev \ + libgmp-dev \ + libicu-dev \ + libidn-dev \ + libpq-dev \ + libssl-dev \ + make \ + shared-mime-info \ + zlib1g-dev \ + ; + +RUN \ +# Configure Corepack + rm /usr/local/bin/yarn*; \ + corepack enable; \ + corepack prepare --activate; + +# Create temporary bundler specific build layer from build layer +FROM build as bundler + +ARG TARGETPLATFORM + +# Copy Gemfile config into working directory +COPY Gemfile* /opt/mastodon/ + +RUN \ +# Mount Ruby Gem caches +--mount=type=cache,id=gem-cache-${TARGETPLATFORM},target=/usr/local/bundle/cache/,sharing=locked \ +# Configure bundle to prevent changes to Gemfile and Gemfile.lock + bundle config set --global frozen "true"; \ +# Configure bundle to not cache downloaded Gems + bundle config set --global cache_all "false"; \ +# Configure bundle to only process production Gems + bundle config set --local without "development test"; \ +# Configure bundle to not warn about root user + bundle config set silence_root_warning "true"; \ +# Download and install required Gems + bundle install -j"$(nproc)"; + +# Create temporary node specific build layer from build layer +FROM build as yarn + +ARG TARGETPLATFORM + +# Copy Node package configuration files into working directory RUN apt-get update && \ apt-get -yq dist-upgrade && \ apt-get install -y --no-install-recommends build-essential \ @@ -41,72 +203,77 @@ COPY Gemfile* package.json yarn.lock .yarnrc.yml /opt/mastodon/ COPY streaming/package.json /opt/mastodon/streaming/ COPY .yarn /opt/mastodon/.yarn -RUN bundle install -j"$(nproc)" +# hadolint ignore=DL3008 +RUN \ +--mount=type=cache,id=corepack-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/corepack,sharing=locked \ +--mount=type=cache,id=yarn-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/yarn,sharing=locked \ +# Install Node packages + yarn workspaces focus --production @mastodon/mastodon; -RUN yarn workspaces focus --all --production && \ - yarn cache clean +# Create temporary assets build layer from build layer +FROM build as precompiler -FROM node:${NODE_VERSION} +# Copy Mastodon sources into precompiler layer +COPY . /opt/mastodon/ -# Use those args to specify your own version flags & suffixes -ARG MASTODON_VERSION_PRERELEASE="" -ARG MASTODON_VERSION_METADATA="" +# Copy bundler and node packages from build layer to container +COPY --from=yarn /opt/mastodon /opt/mastodon/ +COPY --from=bundler /opt/mastodon /opt/mastodon/ +COPY --from=bundler /usr/local/bundle/ /usr/local/bundle/ -ARG UID="991" -ARG GID="991" +ARG TARGETPLATFORM -COPY --link --from=ruby /opt/ruby /opt/ruby +RUN \ +# Use Ruby on Rails to create Mastodon assets + OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder bundle exec rails assets:precompile; \ +# Cleanup temporary files + rm -fr /opt/mastodon/tmp; -SHELL ["/bin/bash", "-o", "pipefail", "-c"] +# Prep final Mastodon Ruby layer +FROM ruby as mastodon -ENV DEBIAN_FRONTEND="noninteractive" \ - PATH="${PATH}:/opt/ruby/bin:/opt/mastodon/bin" +ARG TARGETPLATFORM -# Ignoring these here since we don't want to pin any versions and the Debian image removes apt-get content after use -# hadolint ignore=DL3008,DL3009 -RUN apt-get update && \ - echo "Etc/UTC" > /etc/localtime && \ - groupadd -g "${GID}" mastodon && \ - useradd -l -u "$UID" -g "${GID}" -m -d /opt/mastodon mastodon && \ - apt-get -y --no-install-recommends install whois \ - wget \ - procps \ - libssl3 \ - libpq5 \ - imagemagick \ - ffmpeg \ - libjemalloc2 \ - libicu72 \ - libidn12 \ - libyaml-0-2 \ - file \ - ca-certificates \ - tzdata \ - libreadline8 \ - tini && \ - ln -s /opt/mastodon /mastodon && \ - corepack enable +# hadolint ignore=DL3008 +RUN \ +# Mount Apt cache and lib directories from Docker buildx caches +--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \ +--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \ +# Mount Corepack and Yarn caches from Docker buildx caches +--mount=type=cache,id=corepack-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/corepack,sharing=locked \ +--mount=type=cache,id=yarn-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/yarn,sharing=locked \ +# Apt update install non-dev versions of necessary components + apt-get update; \ + apt-get install -y --no-install-recommends \ + libssl3 \ + libpq5 \ + libicu72 \ + libidn12 \ + libreadline8 \ + libyaml-0-2 \ + ; -# Note: no, cleaning here since Debian does this automatically -# See the file /etc/apt/apt.conf.d/docker-clean within the Docker image's filesystem +# Copy Mastodon sources into final layer +COPY . /opt/mastodon/ -COPY --chown=mastodon:mastodon . /opt/mastodon -COPY --chown=mastodon:mastodon --from=build /opt/mastodon /opt/mastodon +# Copy compiled assets to layer +COPY --from=precompiler /opt/mastodon/public/packs /opt/mastodon/public/packs +COPY --from=precompiler /opt/mastodon/public/assets /opt/mastodon/public/assets +# Copy bundler components to layer +COPY --from=bundler /usr/local/bundle/ /usr/local/bundle/ -ENV RAILS_ENV="production" \ - NODE_ENV="production" \ - RAILS_SERVE_STATIC_FILES="true" \ - BIND="0.0.0.0" \ - MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \ - MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}" +RUN \ +# Precompile bootsnap code for faster Rails startup + bundle exec bootsnap precompile --gemfile app/ lib/; -# Set the run user +RUN \ +# Pre-create and chown system volume to Mastodon user + mkdir -p /opt/mastodon/public/system; \ + chown mastodon:mastodon /opt/mastodon/public/system; + +# Set the running user for resulting container USER mastodon -WORKDIR /opt/mastodon - -# Precompile assets -RUN OTP_SECRET=precompile_placeholder SECRET_KEY_BASE=precompile_placeholder rails assets:precompile - -# Set the work dir and the container entry point -ENTRYPOINT ["/usr/bin/tini", "--"] -EXPOSE 3000 4000 +# Expose default Puma ports +EXPOSE 3000 +# Set container tini as default entry point +ENTRYPOINT ["/usr/bin/tini", "--"] \ No newline at end of file diff --git a/streaming/.dockerignore b/streaming/.dockerignore new file mode 100644 index 00000000000..ea611a7b867 --- /dev/null +++ b/streaming/.dockerignore @@ -0,0 +1,7 @@ +.env +.env.* +.gitignore +node_modules +.DS_Store +*.swp +*~ diff --git a/streaming/Dockerfile b/streaming/Dockerfile new file mode 100644 index 00000000000..ee62e9ec73d --- /dev/null +++ b/streaming/Dockerfile @@ -0,0 +1,102 @@ +# syntax=docker/dockerfile:1.4 + +# Please see https://docs.docker.com/engine/reference/builder for information about +# the extended buildx capabilities used in this file. +# Make sure multiarch TARGETPLATFORM is available for interpolation +# See: https://docs.docker.com/build/building/multi-platform/ +ARG TARGETPLATFORM=${TARGETPLATFORM} +ARG BUILDPLATFORM=${BUILDPLATFORM} + +# Node version to use in base image, change with [--build-arg NODE_MAJOR_VERSION="20"] +ARG NODE_MAJOR_VERSION="20" +# Debian image to use for base image, change with [--build-arg DEBIAN_VERSION="bookworm"] +ARG DEBIAN_VERSION="bookworm" +# Node image to use for base image based on combined variables (ex: 20-bookworm-slim) +FROM docker.io/node:${NODE_MAJOR_VERSION}-${DEBIAN_VERSION}-slim as streaming + +# Timezone used by the Docker container and runtime, change with [--build-arg TZ=Europe/Berlin] +ARG TZ="Etc/UTC" +# Linux UID (user id) for the mastodon user, change with [--build-arg UID=1234] +ARG UID="991" +# Linux GID (group id) for the mastodon user, change with [--build-arg GID=1234] +ARG GID="991" + +# Apply Mastodon build options based on options above +ENV \ +# Apply Mastodon version information + MASTODON_VERSION_PRERELEASE="${MASTODON_VERSION_PRERELEASE}" \ + MASTODON_VERSION_METADATA="${MASTODON_VERSION_METADATA}" \ +# Apply timezone + TZ=${TZ} + +ENV \ +# Configure the IP to bind Mastodon to when serving traffic + BIND="0.0.0.0" \ +# Explicitly set PORT to match the exposed port + PORT=4000 \ +# Use production settings for Yarn, Node and related nodejs based tools + NODE_ENV="production" \ +# Add Ruby and Mastodon installation to the PATH + DEBIAN_FRONTEND="noninteractive" + +# Set default shell used for running commands +SHELL ["/bin/bash", "-o", "pipefail", "-o", "errexit", "-c"] + +ARG TARGETPLATFORM + +RUN echo "Target platform is ${TARGETPLATFORM}" + +RUN \ +# Sets timezone + echo "${TZ}" > /etc/localtime; \ +# Creates mastodon user/group and sets home directory + groupadd -g "${GID}" mastodon; \ + useradd -l -u "${UID}" -g "${GID}" -m -d /opt/mastodon mastodon; \ +# Creates symlink for /mastodon folder + ln -s /opt/mastodon /mastodon; + +# hadolint ignore=DL3008,DL3005 +RUN \ +# Mount Apt cache and lib directories from Docker buildx caches +--mount=type=cache,id=apt-cache-${TARGETPLATFORM},target=/var/cache/apt,sharing=locked \ +--mount=type=cache,id=apt-lib-${TARGETPLATFORM},target=/var/lib/apt,sharing=locked \ +# upgrade to check for security updates to Debian image + apt-get update; \ + apt-get dist-upgrade -yq; \ + apt-get install -y --no-install-recommends \ + ca-certificates \ + curl \ + tzdata \ + ; + +# Set /opt/mastodon as working directory +WORKDIR /opt/mastodon + +# Copy Node package configuration files from build system to container +COPY package.json yarn.lock .yarnrc.yml /opt/mastodon/ +COPY .yarn /opt/mastodon/.yarn +# Copy Streaming source code from build system to container +COPY ./streaming /opt/mastodon/streaming + +RUN \ +# Mount local Corepack and Yarn caches from Docker buildx caches +--mount=type=cache,id=corepack-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/corepack,sharing=locked \ +--mount=type=cache,id=yarn-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/yarn,sharing=locked \ + # Configure Corepack + rm /usr/local/bin/yarn*; \ + corepack enable; \ + corepack prepare --activate; + +RUN \ +# Mount Corepack and Yarn caches from Docker buildx caches +--mount=type=cache,id=corepack-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/corepack,sharing=locked \ +--mount=type=cache,id=yarn-cache-${TARGETPLATFORM},target=/usr/local/share/.cache/yarn,sharing=locked \ +# Install Node packages + yarn workspaces focus --production @mastodon/streaming; + +# Set the running user for resulting container +USER mastodon +# Expose default Streaming ports +EXPOSE 4000 +# Run streaming when started +CMD [ node ./streaming/index.js ] \ No newline at end of file