docker-compose.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

# Usage
# -----
#
# The docker compose file is parametrized using environment variables, the
# defaults are set in .env file.
#
# Example:
# $ ARCH=arm64v8 docker compose build ubuntu-cpp
# $ ARCH=arm64v8 docker compose run ubuntu-cpp
#
#
# Coredumps
# ---------
#
# In order to enable coredumps for the C++ tests run by CTest either with
# command `make unittest` or `ctest --output-on-failure` the correct coredump
# patterns must be set.
# The kernel settings are coming from the host, so while it can be enabled from
# a running container using --privileged option the change will affect all other
# containers, so prefer setting it explicitly, directly on the host.
# WARNING: setting this will affect the host machine.
#
# Linux host:
#   $ sudo sysctl -w kernel.core_pattern=/tmp/core.%e.%p
#
# macOS host running Docker for Mac (won't persist between restarts):
#   $ screen ~/Library/Containers/com.docker.docker/Data/vms/0/tty
#   # echo "/tmp/core.%e.%p" > /proc/sys/kernel/core_pattern
#
# The setup attempts to generate coredumps by default, but the correct paths
# above must be set. In order to disable the coredump generation set
# ULIMIT_CORE environment variable to 0 before running docker compose
# (or by setting it in .env file):
#
# ULIMIT_CORE=0 docker compose run --rm conda-cpp
#
# See more in cpp/build-support/run-test.sh::print_coredumps

x-ccache: &ccache
  CCACHE_COMPILERCHECK: content
  CCACHE_COMPRESS: 1
  CCACHE_COMPRESSLEVEL: 6
  CCACHE_MAXSIZE: 1G
  CCACHE_DIR: /ccache

x-common: &common
  GITHUB_ACTIONS:

x-cpp: &cpp
  ARROW_RUNTIME_SIMD_LEVEL:
  ARROW_SIMD_LEVEL:

x-sccache: &sccache
  AWS_ACCESS_KEY_ID:
  AWS_SECRET_ACCESS_KEY:
  SCCACHE_BUCKET:
  SCCACHE_REGION:
  SCCACHE_S3_KEY_PREFIX: ${SCCACHE_S3_KEY_PREFIX:-sccache}

x-vcpkg-build-secrets: &vcpkg-build-secrets
  - github_repository_owner
  - github_token
  - vcpkg_binary_sources

# CPU/memory limit presets to pass to Docker.
#
# Usage: archery docker run --resource-limit=github <image>
#
# Note that exporting ARCHERY_DOCKER_BIN="sudo docker" is likely required,
# unless Docker is configured with cgroups v2 (else Docker will silently
# ignore the limits).
x-limit-presets:
  # These values emulate GitHub Actions:
  # https://docs.github.com/en/actions/using-github-hosted-runners/about-github-hosted-runners
  github:
    # Note we use cpuset and not cpus since Ninja only detects and limits
    # parallelism given the former
    cpuset_cpus: [0, 1]
    memory: 7g

x-with-gpus:
  - ubuntu-cuda-cpp
  - ubuntu-cuda-python

x-hierarchy:
  # This section is used by the archery tool to enable building nested images,
  # so it is enough to call:
  #   archery docker run debian-ruby
  # instead of a sequence of docker compose commands:
  #   docker compose build debian-cpp
  #   docker compose build debian-c-glib
  #   docker compose build debian-ruby
  #   docker compose run --rm debian-ruby
  #
  # Each node must be either a string scalar of a list containing the
  # descendant images if any. Archery checks that all node has a corresponding
  # service entry, so any new image/service must be listed here.
  - almalinux-verify-rc
  - alpine-linux-cpp
  - centos-cpp-static
  - conda:
    - conda-cpp:
      - conda-integration
      - conda-cpp-valgrind
      - conda-python:
        - conda-python-pandas:
          - conda-python-docs
        - conda-python-cpython-debug
        - conda-python-cython2
        - conda-python-dask
        - conda-python-emscripten
        - conda-python-hdfs
        - conda-python-no-numpy
        - conda-python-spark
        - conda-python-substrait
  - conda-verify-rc
  - conan
  - debian-cpp:
    - debian-c-glib:
      - debian-ruby
    - debian-python:
      - debian-docs
  - debian-js
  - fedora-cpp:
    - fedora-python
  - python-sdist
  - ubuntu-cpp:
    - ubuntu-cpp-static
    - ubuntu-c-glib:
      - ubuntu-ruby
    - ubuntu-lint
    - ubuntu-python
    - ubuntu-python-sdist-test
    - ubuntu-python-313-freethreading
    - ubuntu-r
    - ubuntu-r-only-r
  - ubuntu-cpp-bundled
  - ubuntu-cpp-bundled-offline
  - ubuntu-cpp-minimal
  - ubuntu-cuda-cpp:
    - ubuntu-cuda-python
  - ubuntu-csharp
  - ubuntu-cpp-sanitizer
  - ubuntu-cpp-thread-sanitizer
  - ubuntu-cpp-emscripten
  - ubuntu-r-sanitizer
  - ubuntu-r-valgrind
  - ubuntu-swift
  - ubuntu-verify-rc
  - r-clang-sanitizer
  - r
  - r-revdepcheck
  # helper services
  - impala
  - postgres
  - python-wheel-manylinux-2014
  - python-wheel-manylinux-2-28
  - python-wheel-manylinux-test-imports
  - python-free-threaded-wheel-manylinux-test-imports
  - python-wheel-manylinux-test-unittests
  - python-free-threaded-wheel-manylinux-test-unittests
  - python-wheel-windows-vs2019-base:
    - python-wheel-windows-vs2019
    - python-free-threaded-wheel-windows-vs2019
  - python-wheel-windows-test-base:
    - python-wheel-windows-test
    - python-free-threaded-wheel-windows-test

volumes:
  almalinux-ccache:
    name: ${ARCH}-almalinux-ccache
  alpine-linux-ccache:
    name: ${ARCH}-alpine-linux-ccache
  conda-ccache:
    name: ${ARCH}-conda-ccache
  debian-ccache:
    name: ${ARCH}-debian-${DEBIAN}-ccache
  fedora-ccache:
    name: ${ARCH}-fedora-${FEDORA}-ccache
  maven-cache:
    name: maven-cache
  python-wheel-manylinux2014-ccache:
    name: python-wheel-manylinux2014-ccache
  python-wheel-manylinux-2-28-ccache:
    name: python-wheel-manylinux-2-28-ccache
  ubuntu-ccache:
    name: ${ARCH}-ubuntu-${UBUNTU}-ccache

services:

  ################################# C++ #######################################
  # Release build:
  #   docker compose run -e ARROW_BUILD_TYPE=release conda-cpp|debian-cpp|...
  # Shared only:
  #   docker compose run -e ARROW_BUILD_STATIC=OFF conda-cpp|debian-cpp|...
  # Static only:
  #   docker compose run \
  #     -e ARROW_BUILD_SHARED=OFF \
  #     -e ARROW_TEST_LINKAGE=static \
  #     conda-cpp|debian-cpp|...

  alpine-linux-cpp:
    # Usage:
    #   docker compose build alpine-linux-cpp
    #   docker compose run --rm alpine-linux-cpp
    # Parameters:
    #   ALPINE_LINUX: 3.18
    #   ARCH: amd64, arm64v8, ...
    image: ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp
    build:
      context: .
      dockerfile: ci/docker/alpine-linux-${ALPINE_LINUX}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-alpine-linux-${ALPINE_LINUX}-cpp
      args:
        arch: ${ARCH}
    shm_size: &shm-size 2G
    ulimits: &ulimits
      core: ${ULIMIT_CORE}
    environment:
      <<: [*common, *ccache, *cpp]
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
    volumes: &alpine-linux-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}alpine-linux-ccache:/ccache:delegated
    command: >-
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/cpp_test.sh /arrow /build"

  conda:
    # Base image for conda builds.
    #
    # Usage:
    #   docker compose build conda
    #   docker compose run --rm conda
    # Parameters:
    #   ARCH: amd64, arm32v7
    image: ${REPO}:${ARCH}-conda
    build:
      context: .
      dockerfile: ci/docker/conda.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda
      args:
        arch: ${ARCH}
    volumes:
      - .:/arrow:delegated

  conda-cpp:
    # C++ build in conda environment, including the doxygen docs.
    #
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose run --rm conda-cpp
    # Parameters:
    #   ARCH: amd64, arm32v7
    image: ${REPO}:${ARCH}-conda-cpp
    build:
      context: .
      dockerfile: ci/docker/conda-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-cpp
      args:
        repo: ${REPO}
        arch: ${ARCH}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_BUILD_BENCHMARKS: "ON"
      ARROW_BUILD_EXAMPLES: "ON"
      ARROW_BUILD_OPENMP_BENCHMARKS: "ON"
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_EXTRA_ERROR_CONTEXT: "ON"
      ARROW_MIMALLOC: "ON"
      ARROW_USE_LD_GOLD: "ON"
      BUILD_DOCS_PYTHON: "ON"
    volumes: &conda-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated
    command: &conda-cpp-command
      ["
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/cpp_test.sh /arrow /build"]

  conda-cpp-valgrind:
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose run --rm conda-cpp-valgrind
    # Parameters:
    #   ARCH: amd64, arm32v7
    image: ${REPO}:${ARCH}-conda-cpp
    build:
      context: .
      dockerfile: ci/docker/conda-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-cpp
      args:
        repo: ${REPO}
        arch: ${ARCH}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      # Shrink test runtime by enabling minimal optimizations
      ARROW_C_FLAGS_DEBUG: "-g1 -Og"
      ARROW_CXX_FLAGS_DEBUG: "-g1 -Og"
      # GH-39973: Do not use debug memory pool for valgrind
      ARROW_DEBUG_MEMORY_POOL: "none"
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_FLIGHT: "OFF"
      ARROW_FLIGHT_SQL: "OFF"
      ARROW_GANDIVA: "OFF"
      ARROW_JEMALLOC: "OFF"
      ARROW_RUNTIME_SIMD_LEVEL: "AVX2"  # AVX512 not supported by Valgrind (ARROW-9851)
      ARROW_TEST_MEMCHECK: "ON"
      ARROW_USE_LD_GOLD: "ON"
      BUILD_WARNING_LEVEL: "PRODUCTION"
      ARROW_CTEST_TIMEOUT: 500
    volumes: *conda-volumes
    command: *conda-cpp-command

  debian-cpp:
    # Usage:
    #   docker compose build debian-cpp
    #   docker compose run --rm debian-cpp
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 12, experimental
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
    build:
      context: .
      dockerfile: ci/docker/debian-${DEBIAN}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
      args:
        arch: ${ARCH}
        gcc: ${GCC}
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
    volumes: &debian-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}debian-ccache:/ccache:delegated
    # integration_skyhook.sh is a no-op unless skyhook is on.
    command: &cpp-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_skyhook.sh /build &&
        /arrow/ci/scripts/cpp_test.sh /arrow /build"

  ubuntu-cpp:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose run --rm ubuntu-cpp
    # Parameters:
    #   ARCH: amd64, arm64v8, s390x, ...
    #   UBUNTU: 20.04, 22.04, 24.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
        clang_tools: ${CLANG_TOOLS}
        gcc: ${GCC}
        llvm: ${LLVM}
    shm_size: *shm-size
    cap_add:
      - SYS_ADMIN
    devices:
      - "/dev/fuse:/dev/fuse"
    security_opt:
      - apparmor:unconfined
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
    volumes: &ubuntu-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated
    command: *cpp-command

  ubuntu-cpp-static:
    # Usage:
    #   docker compose build ubuntu-cpp-static
    #   docker compose run --rm ubuntu-cpp-static
    # Parameters:
    #   ARCH: amd64, arm64v8, s390x, ...
    #   UBUNTU: 20.04, 22.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-static
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
        clang_tools: ${CLANG_TOOLS}
        gcc: ${GCC}
        llvm: ${LLVM}
    shm_size: *shm-size
    cap_add:
      - SYS_ADMIN
    devices:
      - "/dev/fuse:/dev/fuse"
    security_opt:
      - apparmor:unconfined
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_HOME: /arrow
      ARROW_DEPENDENCY_SOURCE: BUNDLED
      LIBARROW_MINIMAL: "false"
      ARROW_MIMALLOC: "ON"
    volumes: *ubuntu-volumes
    command: &cpp-static-command
      /bin/bash -c "
        cd /arrow &&
        r/inst/build_arrow_static.sh"

  centos-cpp-static:
    image: ${REPO}:centos-7-cpp-static
    build:
      context: .
      dockerfile: ci/docker/centos-7-cpp.dockerfile
      cache_from:
        - ${REPO}:centos-7-cpp-static
    shm_size: *shm-size
    volumes:
      - .:/arrow:delegated
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_DEPENDENCY_SOURCE: BUNDLED
      ARROW_HOME: /arrow
      LIBARROW_MINIMAL: "false"
    command: /bin/bash -c "
          scl enable devtoolset-8 '/arrow/r/inst/build_arrow_static.sh'"

  ubuntu-cpp-bundled:
    # Arrow build with BUNDLED dependencies
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_DEPENDENCY_SOURCE: BUNDLED
      CMAKE_GENERATOR: "Unix Makefiles"
    volumes: *ubuntu-volumes
    command: *cpp-command

  ubuntu-cpp-bundled-offline:
    # Arrow build with BUNDLED dependencies with downloaded dependencies.
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *cpp]
      ARROW_DEPENDENCY_SOURCE: BUNDLED
      ARROW_OFFLINE: ON
      # Apache ORC always uses external orc-format.
      ARROW_ORC: OFF
      CMAKE_GENERATOR: "Unix Makefiles"
    volumes: *ubuntu-volumes
    command: *cpp-command

  ubuntu-cpp-minimal:
    # Arrow build with minimal components/dependencies
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp-minimal.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp-minimal
      args:
        arch: ${ARCH}
        base: "${ARCH}/ubuntu:${UBUNTU}"
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_BUILD_UTILITIES: "OFF"
      ARROW_ACERO: "OFF"
      ARROW_AZURE: "OFF"
      ARROW_COMPUTE: "OFF"
      ARROW_CSV: "OFF"
      ARROW_DATASET: "OFF"
      ARROW_FILESYSTEM: "OFF"
      ARROW_FLIGHT: "OFF"
      ARROW_GANDIVA: "OFF"
      ARROW_GCS: "OFF"
      ARROW_HDFS: "OFF"
      ARROW_ORC: "OFF"
      ARROW_PARQUET: "OFF"
      ARROW_S3: "OFF"
      ARROW_SUBSTRAIT: "OFF"
      ARROW_WITH_BROTLI: "OFF"
      ARROW_WITH_BZ2: "OFF"
      ARROW_WITH_LZ4: "OFF"
      ARROW_WITH_SNAPPY: "OFF"
      ARROW_WITH_ZLIB: "OFF"
      ARROW_WITH_ZSTD: "OFF"
      PARQUET_BUILD_EXAMPLES: "OFF"
      PARQUET_BUILD_EXECUTABLES: "OFF"
      PARQUET_REQUIRE_ENCRYPTION: "OFF"
    volumes: *ubuntu-volumes
    command: *cpp-command

  ubuntu-cuda-cpp:
    # Usage:
    #   docker compose build cuda-cpp
    #   docker compose run --rm cuda-cpp
    # Parameters:
    #   ARCH: amd64
    #   CUDA: <depends on your nvidia driver, should match system CUDA>
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
      args:
        arch: ${ARCH}
        base: nvidia/cuda:${CUDA}-devel-ubuntu${UBUNTU}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_BUILD_UTILITIES: "OFF"
      ARROW_COMPUTE: "OFF"
      ARROW_CSV: "OFF"
      ARROW_CUDA: "ON"
      ARROW_DATASET: "OFF"
      ARROW_ENABLE_TIMING_TESTS: "OFF"
      ARROW_FILESYSTEM: "OFF"
      ARROW_GANDIVA: "OFF"
      ARROW_GCS: "OFF"
      ARROW_HDFS: "OFF"
      ARROW_JEMALLOC: "OFF"
      ARROW_JSON: "OFF"
      ARROW_ORC: "OFF"
      ARROW_PARQUET: "OFF"
      ARROW_S3: "OFF"
      ARROW_SUBSTRAIT: "OFF"
      ARROW_WITH_OPENTELEMETRY: "OFF"
    volumes: *ubuntu-volumes
    command: *cpp-command
    deploy: &cuda-deploy
      resources:
        reservations:
          devices:
            - driver: nvidia
              capabilities: [gpu]

  ubuntu-cpp-sanitizer:
    # Usage:
    #   docker compose build ubuntu-cpp-sanitizer
    #   docker compose run --rm ubuntu-cpp-sanitizer
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 22.04, ...
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    cap_add:
      # For LeakSanitizer
      - SYS_PTRACE
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    environment:
      <<: [*common, *ccache, *cpp]
      CC: clang-${CLANG_TOOLS}
      CXX: clang++-${CLANG_TOOLS}
      # Avoid creating huge static libraries
      ARROW_BUILD_STATIC: "OFF"
      # GH-39973: Do not use debug memory pool for ASAN
      ARROW_DEBUG_MEMORY_POOL: "none"
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      # GH-33920: Disable Flight SQL to reduce build time.
      # We'll be able to re-enable this with Ubuntu 24.04 because
      # Ubuntu 24.04 will ship ProtoBuf 3.15.0 or later that is required
      # by Flight SQL.
      ARROW_FLIGHT_SQL: "OFF"
      ARROW_FUZZING: "ON"  # Check fuzz regressions
      ARROW_JEMALLOC: "OFF"
      ARROW_MIMALLOC: "OFF"
      ARROW_ORC: "OFF"
      ARROW_S3: "OFF"
      ARROW_USE_ASAN: "ON"
      ARROW_USE_UBSAN: "ON"
      # 1 GB isn't enough for a single sanitizer build
      CCACHE_MAXSIZE: 2G
      Protobuf_SOURCE: "AUTO"
    command: *cpp-command

  ubuntu-cpp-thread-sanitizer:
    # Usage:
    #   docker compose build ubuntu-cpp-thread-sanitizer
    #   docker compose run --rm ubuntu-cpp-thread-sanitizer
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 20.04, 22.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      CC: clang-${CLANG_TOOLS}
      CXX: clang++-${CLANG_TOOLS}
      ARROW_BUILD_STATIC: "OFF"
      ARROW_CTEST_TIMEOUT: 500
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_FLIGHT: "OFF"
      ARROW_FLIGHT_SQL: "OFF"
      ARROW_JEMALLOC: "OFF"
      ARROW_MIMALLOC: "OFF"
      ARROW_ORC: "OFF"
      ARROW_USE_TSAN: "ON"
    command: *cpp-command

  ubuntu-cpp-emscripten:
    # Usage:
    #   docker compose build ubuntu-cpp-emscripten
    #   docker compose run --rm ubuntu-cpp-emscripten
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 22.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
      args:
        arch: ${ARCH}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_EMSCRIPTEN: "ON"
      UBUNTU:
    command: *cpp-command

  fedora-cpp:
    # Usage:
    #   docker compose build fedora-cpp
    #   docker compose run --rm fedora-cpp
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   FEDORA: 39
    image: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
    build:
      context: .
      dockerfile: ci/docker/fedora-${FEDORA}-cpp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
      args:
        arch: ${ARCH}
        llvm: ${LLVM}
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
      ARROW_ENABLE_TIMING_TESTS:  # inherit
      ARROW_MIMALLOC: "ON"
    volumes: &fedora-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}fedora-ccache:/ccache:delegated
    command: *cpp-command

  conan:
    # Base service for Conan.
    #
    # Usage:
    #   docker compose run --rm conan
    # Parameters:
    #   CONAN_BASE: gcc11, gcc11-armv7, ...
    #   CONAN_VERSION: 1.62.0
    #   See https://github.com/conan-io/conan-docker-tools#readme for
    #   available images.
    image: conanio/${CONAN_BASE}:${CONAN_VERSION}
    user: root:root
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *sccache]
    volumes:
      - .:/arrow:delegated
    command: >-
      /bin/bash -c "
        sudo /arrow/ci/scripts/install_sccache.sh unknown-linux-musl /usr/local/bin &&
        /arrow/ci/scripts/conan_setup.sh &&
        /arrow/ci/scripts/conan_build.sh /arrow /build"

  ############################### C GLib ######################################

  debian-c-glib:
    # Usage:
    #   docker compose build debian-cpp
    #   docker compose build debian-c-glib
    #   docker compose run --rm debian-c-glib
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 12
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
    build:
      context: .
      dockerfile: ci/docker/linux-apt-c-glib.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache]
      BUILD_DOCS_C_GLIB: "ON"
    volumes: *debian-volumes
    command: &c-glib-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_test.sh /arrow /build"

  ubuntu-c-glib:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-c-glib
    #   docker compose run --rm ubuntu-c-glib
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 20.04, 22.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
    build:
      context: .
      dockerfile: ci/docker/linux-apt-c-glib.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache]
    volumes: *ubuntu-volumes
    command: *c-glib-command

  ############################### Ruby ########################################
  # Until Ruby is the only dependent implementation on top of C Glib we can
  # test C Glib and Ruby in one pass. This is an optimization to avoid
  # redundant (one for C GLib and one for Ruby doing the same work twice)
  # builds on CI services.

  debian-ruby:
    # Usage:
    #   docker compose build debian-cpp
    #   docker compose build debian-c-glib
    #   docker compose build debian-ruby
    #   docker compose run --rm debian-ruby
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 12
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-ruby
    build:
      context: .
      dockerfile: ci/docker/linux-apt-ruby.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-ruby
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-c-glib
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache]
      BUILD_DOCS_C_GLIB: "ON"
    volumes: *debian-volumes
    command: &ruby-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_test.sh /arrow /build &&
        /arrow/ci/scripts/ruby_test.sh /arrow /build"

  ubuntu-ruby:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-c-glib
    #   docker compose build ubuntu-ruby
    #   docker compose run --rm ubuntu-ruby
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 20.04, 22.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby
    build:
      context: .
      dockerfile: ci/docker/linux-apt-ruby.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-ruby
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-c-glib
    shm_size: *shm-size
    ulimits: *ulimits
    environment:
      <<: [*common, *ccache]
    volumes: *ubuntu-volumes
    command: *ruby-command

  ############################### Python ######################################

  conda-python:
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose run --rm conda-python
    # Parameters:
    #   ARCH: amd64, arm32v7
    #   PYTHON: 3.9, 3.10, 3.11, 3.12
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}
    build:
      context: .
      dockerfile: ci/docker/conda-python.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache, *sccache]
      PYTEST_ARGS:  # inherit
    volumes: *conda-volumes
    command: &python-conda-command
      ["
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/python_test.sh /arrow"]

  conda-python-emscripten:
    # Usage:
    #   docker compose build conda-python-emscripten
    #   docker compose run --rm conda-python-emscripten
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 22.04
    image: ${REPO}:${ARCH}-conda-python-emscripten
    build:
      context: .
      dockerfile: ci/docker/conda-python-emscripten.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        clang_tools: ${CLANG_TOOLS}
        llvm: ${LLVM}
        pyodide_version: "0.26.0"
        chrome_version: "122"
        selenium_version: "4.15.2"
        required_python_min: "(3,12)"
        python: ${PYTHON}
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    environment:
      <<: [*common, *ccache, *sccache, *cpp]
    command: ["
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build_emscripten.sh /arrow /build &&
        /arrow/ci/scripts/python_test_emscripten.sh /build /pyodide"]

  ubuntu-cuda-python:
    # Usage:
    #   docker compose build cuda-cpp
    #   docker compose build cuda-python
    #   docker compose run --rm cuda-python
    # Parameters:
    #   ARCH: amd64
    #   CUDA: <depends on your nvidia driver, should match system CUDA>
    #   UBUNTU: 22.04, 24.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-python-3
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cuda-${CUDA}-cpp
        numba: ${NUMBA}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache, *sccache]
      ARROW_BUILD_UTILITIES: "OFF"
      ARROW_COMPUTE: "ON"
      ARROW_CSV: "ON"
      ARROW_CUDA: "ON"
      ARROW_DATASET: "ON"
      ARROW_ENABLE_TIMING_TESTS: "OFF"
      ARROW_FILESYSTEM: "ON"
      ARROW_GANDIVA: "OFF"
      ARROW_GCS: "OFF"
      ARROW_HDFS: "ON"
      ARROW_JEMALLOC: "ON"
      ARROW_JSON: "ON"
      ARROW_ORC: "OFF"
      ARROW_PARQUET: "ON"
      ARROW_S3: "OFF"
      ARROW_SUBSTRAIT: "OFF"
      ARROW_WITH_OPENTELEMETRY: "OFF"
      SETUPTOOLS_SCM_PRETEND_VERSION:
    volumes: *ubuntu-volumes
    deploy: *cuda-deploy
    command: &python-command >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/python_test.sh /arrow"

  debian-python:
    # Usage:
    #   docker compose build debian-cpp
    #   docker compose build debian-python
    #   docker compose run --rm debian-python
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   DEBIAN: 12
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-python-3
      args:
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-cpp
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
    volumes: *debian-volumes
    command: *python-command

  ubuntu-python:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-python
    #   docker compose run --rm ubuntu-python
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 22.04, 24.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
    volumes: *ubuntu-volumes
    command: *python-command

  ubuntu-swift:
    # Usage:
    #   docker compose build ubuntu-swift
    #   docker compose run --rm ubuntu-swift
    # Parameters:
    image: ubuntu-swift
    build:
      context: .
      dockerfile: ci/docker/ubuntu-swift.dockerfile
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "/arrow/ci/scripts/swift_test.sh /arrow"


  fedora-python:
    # Usage:
    #   docker compose build fedora-cpp
    #   docker compose build fedora-python
    #   docker compose run --rm fedora-python
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   FEDORA: 39
    image: ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-dnf-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-fedora-${FEDORA}-python-3
      args:
        base: ${REPO}:${ARCH}-fedora-${FEDORA}-cpp
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
    volumes: *fedora-volumes
    command: *python-command

  ############################ Python sdist ###################################

  python-sdist:
    # Usage:
    #   docker compose build python-sdist
    #   docker compose run --rm python-sdist
    # Parameters:
    #   PYARROW_VERSION: The pyarrow version for sdist such as "3.0.0"
    image: ${REPO}:python-sdist
    build:
      context: .
      dockerfile: ci/docker/python-sdist.dockerfile
      cache_from:
        - ${REPO}:python-sdist
    environment:
      PYARROW_VERSION: ${PYARROW_VERSION:-}
    volumes:
      - .:/arrow:delegated
    command: /arrow/ci/scripts/python_sdist_build.sh /arrow

  ubuntu-python-sdist-test:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-python-sdist-test
    #   docker compose run --rm ubuntu-python-sdist-test
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   PYARROW_VERSION: The test target pyarrow version such as "3.0.0"
    #   UBUNTU: 22.04, 24.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-3.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-3
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
      # Bundled build of OpenTelemetry needs a git client
      ARROW_WITH_OPENTELEMETRY: "OFF"
      PYARROW_VERSION: ${PYARROW_VERSION:-}
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        apt remove -y git &&
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_sdist_test.sh /arrow"

  ############################ Python free-threading ##########################

  ubuntu-python-313-freethreading:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-python-313-freethreading
    #   docker compose run --rm ubuntu-python-313-freethreading
    # Parameters:
    #   ARCH: amd64, arm64v8, ...
    #   UBUNTU: 20.04, 22.04, 24.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading
    build:
      context: .
      dockerfile: ci/docker/linux-apt-python-313-freethreading.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-python-313-freethreading
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
      # Bundled build of OpenTelemetry needs a git client
      ARROW_WITH_OPENTELEMETRY: "OFF"
    volumes: *ubuntu-volumes
    command: *python-command

  ############################ Python wheels ##################################

  # See available versions at:
  #    https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags
  python-wheel-manylinux-2014:
    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
    build:
      args:
        arch: ${ARCH}
        arch_short: ${ARCH_SHORT}
        base: quay.io/pypa/manylinux2014_${ARCH_ALIAS}:2024-08-03-32dfa47
        manylinux: 2014
        python: ${PYTHON}
        python_abi_tag: ${PYTHON_ABI_TAG}
        vcpkg: ${VCPKG}
      context: .
      dockerfile: ci/docker/python-wheel-manylinux.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2014-vcpkg-${VCPKG}
      secrets: *vcpkg-build-secrets
    environment:
      <<: [*common, *ccache]
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux2014-ccache:/ccache:delegated
    command: /arrow/ci/scripts/python_wheel_manylinux_build.sh

  # See available versions at:
  #    https://quay.io/repository/pypa/manylinux_2_28_x86_64?tab=tags
  python-wheel-manylinux-2-28:
    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
    build:
      args:
        arch: ${ARCH}
        arch_short: ${ARCH_SHORT}
        base: quay.io/pypa/manylinux_2_28_${ARCH_ALIAS}:2024-08-03-32dfa47
        manylinux: 2_28
        python: ${PYTHON}
        python_abi_tag: ${PYTHON_ABI_TAG}
        vcpkg: ${VCPKG}
      context: .
      dockerfile: ci/docker/python-wheel-manylinux.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-2-28-vcpkg-${VCPKG}
      secrets: *vcpkg-build-secrets
    environment:
      <<: [*common, *ccache]
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}python-wheel-manylinux-2-28-ccache:/ccache:delegated
    command: /arrow/ci/scripts/python_wheel_manylinux_build.sh

  python-wheel-manylinux-test-imports:
    image: ${ARCH}/python:${PYTHON_IMAGE_TAG}
    shm_size: 2G
    volumes:
      - .:/arrow:delegated
    environment:
      <<: *common
      CHECK_IMPORTS: "ON"
      CHECK_UNITTESTS: "OFF"
      CHECK_WHEEL_CONTENT: "ON"
    command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

  # TODO: Remove this when the official Docker Python image supports the free-threaded build.
  # See https://github.com/docker-library/python/issues/947 for more info.
  python-free-threaded-wheel-manylinux-test-imports:
    image: ${REPO}:${ARCH}-python-3.13-free-threaded-wheel-manylinux-test-imports
    build:
      args:
        base: "${ARCH}/ubuntu:${UBUNTU}"
      context: .
      dockerfile: ci/docker/python-free-threaded-wheel-manylinux-test-imports.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-3.13-free-threaded-wheel-manylinux-test-imports
    shm_size: 2G
    volumes:
      - .:/arrow:delegated
    environment:
      <<: *common
      CHECK_IMPORTS: "ON"
      CHECK_UNITTESTS: "OFF"
    command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

  python-wheel-manylinux-test-unittests:
    image: ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test
    build:
      args:
        arch: ${ARCH}
        python: ${PYTHON}
        python_image_tag: ${PYTHON_IMAGE_TAG}
      context: .
      dockerfile: ci/docker/python-wheel-manylinux-test.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-${PYTHON}-wheel-manylinux-test
    shm_size: 2G
    volumes:
      - .:/arrow:delegated
    environment:
      <<: *common
      CHECK_IMPORTS: "OFF"
      CHECK_UNITTESTS: "ON"
      CHECK_WHEEL_CONTENT: "OFF"
    command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

  # TODO: Remove this when the official Docker Python image supports the free-threaded build.
  # See https://github.com/docker-library/python/issues/947 for more info.
  python-free-threaded-wheel-manylinux-test-unittests:
    image: ${REPO}:${ARCH}-python-3.13-free-threaded-wheel-manylinux-test-unittests
    build:
      args:
        base: "${ARCH}/ubuntu:${UBUNTU}"
      context: .
      dockerfile: ci/docker/python-free-threaded-wheel-manylinux-test-unittests.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-python-3.13-free-threaded-wheel-manylinux-test-unittests
    shm_size: 2G
    volumes:
      - .:/arrow:delegated
    environment:
      <<: *common
      CHECK_IMPORTS: "OFF"
      CHECK_UNITTESTS: "ON"
    command: /arrow/ci/scripts/python_wheel_unix_test.sh /arrow

  python-wheel-windows-vs2019-base:
    image: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
    build:
      args:
        vcpkg: ${VCPKG}
      context: .
      dockerfile: ci/docker/python-wheel-windows-vs2019-base.dockerfile
      # This should make the pushed images reusable, but the image gets rebuilt.
      # Uncomment if no local cache is available.
      # cache_from:
      #   - abrarov/msvc-2019:2.11.0
      #   - ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
    volumes: &python-wheel-windows-vs2019-volumes
      - type: bind
        source: .
        target: "C:/arrow"

  python-wheel-windows-vs2019:
    image: ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
    build:
      args:
        base: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
        python: ${PYTHON}
      context: .
      dockerfile: ci/docker/python-wheel-windows-vs2019.dockerfile
      # This should make the pushed images reusable, but the image gets rebuilt.
      # Uncomment if no local cache is available.
      # cache_from:
      #   - abrarov/msvc-2019:2.11.0
      #   - ${REPO}:python-${PYTHON}-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
    volumes: *python-wheel-windows-vs2019-volumes
    command: arrow\\ci\\scripts\\python_wheel_windows_build.bat

  python-free-threaded-wheel-windows-vs2019:
    image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
    build:
      args:
        base: ${REPO}:python-wheel-windows-vs2019-base-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
        python: ${PYTHON}
      context: .
      dockerfile: ci/docker/python-free-threaded-wheel-windows-vs2019.dockerfile
      # This should make the pushed images reusable, but the image gets rebuilt.
      # Uncomment if no local cache is available.
      # cache_from:
      #   - abrarov/msvc-2019:2.11.0
      #   - ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-vs2019-vcpkg-${VCPKG}-${PYTHON_WHEEL_WINDOWS_IMAGE_REVISION}
    volumes: *python-wheel-windows-vs2019-volumes
    command: arrow\\ci\\scripts\\python_wheel_windows_build.bat

  python-wheel-windows-test-base:
    image: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
    build:
      context: .
      dockerfile: ci/docker/python-wheel-windows-test-vs2019-base.dockerfile
    volumes: *python-wheel-windows-vs2019-volumes

  python-wheel-windows-test:
    image: ${REPO}:python-${PYTHON}-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
    build:
      args:
        base: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
        python: ${PYTHON}
      context: .
      dockerfile: ci/docker/python-wheel-windows-test-vs2019.dockerfile
    volumes: *python-wheel-windows-vs2019-volumes
    command: arrow\\ci\\scripts\\python_wheel_windows_test.bat

  python-free-threaded-wheel-windows-test:
    image: ${REPO}:python-${PYTHON}-free-threaded-wheel-windows-test-vs2019-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
    build:
      args:
        base: ${REPO}:python-wheel-windows-test-vs2019-base-${PYTHON_WHEEL_WINDOWS_TEST_IMAGE_REVISION}
      context: .
      dockerfile: ci/docker/python-free-threaded-wheel-windows-test-vs2019.dockerfile
    volumes: *python-wheel-windows-vs2019-volumes
    command: arrow\\ci\\scripts\\python_wheel_windows_test.bat

  ##############################  Integration #################################

  conda-python-pandas:
    # Possible $PANDAS parameters:
    #  - `latest`: latest release
    #  - `master`: git master branch, use `docker compose run --no-cache`
    #  - `<version>`: specific version available on conda-forge
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-pandas
    #   docker compose run --rm conda-python-pandas
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS}
    build:
      context: .
      dockerfile: ci/docker/conda-python-pandas.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        numpy: ${NUMPY}
        pandas: ${PANDAS}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache, *sccache]
      PARQUET_REQUIRE_ENCRYPTION:  # inherit
      PYTEST_ARGS:  # inherit
      HYPOTHESIS_PROFILE:  # inherit
      PYARROW_TEST_HYPOTHESIS:  # inherit
      PANDAS_FUTURE_INFER_STRING:  # inherit
    volumes: *conda-volumes
    command: *python-conda-command

  conda-python-no-numpy:
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-no-numpy
    #   docker compose run --rm conda-python-no-numpy
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-no-numpy
    build:
      context: .
      dockerfile: ci/docker/conda-python.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache, *sccache]
      PARQUET_REQUIRE_ENCRYPTION:  # inherit
      HYPOTHESIS_PROFILE:  # inherit
      PYARROW_TEST_HYPOTHESIS:  # inherit
    volumes: *conda-volumes
    command:
      ["
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        mamba uninstall -y numpy &&
        /arrow/ci/scripts/python_test.sh /arrow"]

  conda-python-docs:
    # Usage:
    #   archery docker run conda-python-docs
    #
    # Only a single rule is enabled for now to check undocumented arguments.
    # We should extend the list of enabled rules after adding this build to
    # the CI pipeline.
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-pandas-${PANDAS}
    cap_add:
      - SYS_ADMIN
    environment:
      <<: [*common, *ccache]
      ARROW_SUBSTRAIT: "ON"
      LC_ALL: "C.UTF-8"
      LANG: "C.UTF-8"
      BUILD_DOCS_CPP: "ON"
      BUILD_DOCS_PYTHON: "ON"
      PYTEST_ARGS: "--doctest-modules --doctest-cython"
    volumes: *conda-volumes
    # pytest is installed with an upper pin of 8.0.0 because
    # newer version breaks cython doctesting, see:
    # https://github.com/lgpage/pytest-cython/issues/58
    # Remove pip install pytest~=7 when upstream issue is resolved
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        pip install -e /arrow/dev/archery[numpydoc] &&
        pip install pytest~=7.4 &&
        archery numpydoc --allow-rule GL10,PR01,PR03,PR04,PR05,PR10,RT03,YD01 &&
        /arrow/ci/scripts/python_test.sh /arrow"]

  conda-python-dask:
    # Possible $DASK parameters:
    #  - `latest`: latest release
    #  - `master`: git master branch, use `docker compose run --no-cache`
    #  - `<version>`: specific version available on conda-forge
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-dask
    #   docker compose run --rm conda-python-dask
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK}
    build:
      context: .
      dockerfile: ci/docker/conda-python-dask.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-dask-${DASK}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        dask: ${DASK}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_dask.sh"]

  conda-python-substrait:
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-substrait
    #   docker compose run --rm conda-python-substrait
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}
    build:
      context: .
      dockerfile: ci/docker/conda-python-substrait.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
      ARROW_SUBSTRAIT: "ON"
    volumes: *conda-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_substrait.sh"]

  conda-python-cython2:
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-cython2
    #   docker compose run --rm conda-python-cython2
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2
    build:
      context: .
      dockerfile: ci/docker/conda-python-cython2.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-cython2
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
      PYTEST_ARGS:  # inherit
    volumes: *conda-volumes
    command: *python-conda-command

  conda-python-cpython-debug:
    # Usage:
    #   docker compose build conda
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-cpython-debug
    #   docker compose run --rm conda-python-cpython-debug
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug
    build:
      context: .
      dockerfile: ci/docker/conda-python-cpython-debug.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-cpython-debug
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
      PYTEST_ARGS:  # inherit
    volumes: *conda-volumes
    command: *python-conda-command

  ################################## R ########################################

  ubuntu-r:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-r
    #   docker compose run ubuntu-r
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R}
    build:
      context: .
      dockerfile: ci/docker/linux-apt-r.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-r-${R}
      args:
        arch: ${ARCH}
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
        gcc: ${GCC}
        r: ${R}
        r_duckdb_dev: ${R_DUCKDB_DEV:-}
        r_prune_deps: ${R_PRUNE_DEPS}
        tz: ${TZ}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache, *sccache]
      ARROW_R_CXXFLAGS: '-Werror'
      ARROW_FLIGHT: 'ON'
      LIBARROW_BUILD: 'false'
      NOT_CRAN: 'true'
      ARROW_R_DEV: ${ARROW_R_DEV}
      ARROW_SOURCE_HOME: '/arrow'
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/r_test.sh /arrow"

  ubuntu-r-only-r:
    environment:
      <<: *common
      ARROW_DEPENDENCY_SOURCE: ''
      ARROW_SOURCE_HOME: '/arrow'
      FORCE_BUNDLED_BUILD: 'true'
      LIBARROW_BINARY: 'false'
      LIBARROW_BUILD: 'true'
    extends: ubuntu-r
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_test.sh /arrow"

  r:
    # This lets you test building/installing the arrow R package
    # (including building the C++ library) on any Docker image that contains R
    #
    # Usage:
    #   R_ORG=rhub R_IMAGE=ubuntu-release R_TAG=latest docker compose build r
    #   R_ORG=rhub R_IMAGE=ubuntu-release R_TAG=latest docker compose run r
    image: ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG}
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:r-${R_ORG}-${R_IMAGE}-${R_TAG}
      args:
        base: ${R_ORG}/${R_IMAGE}:${R_TAG}
        r_dev: ${ARROW_R_DEV}
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
        r_custom_ccache: ${R_CUSTOM_CCACHE}
    shm_size: *shm-size
    environment:
      <<: [*common, *sccache]
      LIBARROW_BINARY: "false"
      ARROW_SOURCE_HOME: "/arrow"
      ARROW_R_DEV: ${ARROW_R_DEV}
      # To test for CRAN release, delete ^^ these two env vars so we download the Apache release
      ARROW_USE_PKG_CONFIG: "false"
    volumes:
      - .:/arrow:delegated
    command: >
      /bin/bash -c "/arrow/ci/scripts/r_test.sh /arrow"

  ubuntu-r-sanitizer:
    # Only 20.04 and amd64 supported
    # Usage:
    #   docker compose build ubuntu-r-sanitizer
    #   docker compose run ubuntu-r-sanitizer
    image: ${REPO}:amd64-ubuntu-20.04-r-sanitizer
    cap_add:
      # LeakSanitizer and gdb requires ptrace(2)
      - SYS_PTRACE
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:amd64-ubuntu-20.04-r-sanitizer
      args:
        base: wch1/r-debug:latest
        r_bin: RDsan
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
    environment:
      <<: [*common, *ccache]
      ARROW_SOURCE_HOME: "/arrow"
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_sanitize.sh /arrow"

  r-clang-sanitizer:
    image: ${REPO}:r-rhub-clang-devel-latest
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:r-rhub-clang-devel-latest
      args:
        base: rhub/clang-asan
        r_dev: ${ARROW_R_DEV}
        r_bin: R
        tz: ${TZ}
        r_prune_deps: ${R_PRUNE_DEPS}
    shm_size: *shm-size
    environment:
      <<: *common
      LIBARROW_DOWNLOAD: "false"
      ARROW_SOURCE_HOME: "/arrow"
      ARROW_R_DEV: ${ARROW_R_DEV}
      # To test for CRAN release, delete ^^ these two env vars so we download the Apache release
      ARROW_USE_PKG_CONFIG: "false"
    volumes:
      - .:/arrow:delegated
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_sanitize.sh /arrow"

  ubuntu-r-valgrind:
    # Only 20.04 and amd64 supported
    # Usage:
    #   docker compose build ubuntu-r-valgrind
    #   docker compose run ubuntu-r-valgrind
    image: ${REPO}:amd64-ubuntu-20.04-r-valgrind
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:amd64-ubuntu-20.04-r-valgrind
      args:
        base: wch1/r-debug:latest
        r_bin: RDvalgrind
        tz: ${TZ}
    environment:
      <<: [*common, *ccache, *sccache]
      ARROW_R_DEV: ${ARROW_R_DEV}
      # AVX512 not supported by Valgrind (similar to ARROW-9851) some runners support AVX512 and some do not
      # so some build might pass without this setting, but we want to ensure that we stay to AVX2 regardless of runner.
      EXTRA_CMAKE_FLAGS: "-DARROW_RUNTIME_SIMD_LEVEL=AVX2"
      ARROW_SOURCE_HOME: "/arrow"
      LIBARROW_DOWNLOAD: "false"
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        /arrow/ci/scripts/r_valgrind.sh /arrow"

  r-revdepcheck:
    # Usage:
    #   docker compose build r-revdepcheck
    #   docker compose run r-revdepcheck
    image: ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck
    build:
      context: .
      dockerfile: ci/docker/linux-r.dockerfile
      cache_from:
        - ${REPO}:r-rstudio-r-base-4.2-focal-revdepcheck
      args:
        base: rstudio/r-base:4.2-focal
        r_dev: ${ARROW_R_DEV}
        tz: ${TZ}
    shm_size: *shm-size
    environment:
      <<: *common
      N_JOBS:
      ARROW_REVDEP_WORKERS:
      ARROW_R_DEV: "true"
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "/arrow/ci/scripts/r_revdepcheck.sh /arrow"

  ############################# JavaScript ####################################

  debian-js:
    # Usage:
    #   docker compose build debian-js
    #   docker compose run debian-js
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE}
    build:
      context: .
      dockerfile: ci/docker/debian-${DEBIAN}-js.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-js-${NODE}
      args:
        arch: ${ARCH}
        node: ${NODE}
    shm_size: *shm-size
    volumes: *debian-volumes
    environment:
      <<: *common
      BUILD_DOCS_JS: "ON"
    command: &js-command >
      /bin/bash -c "
        /arrow/ci/scripts/js_build.sh /arrow /build &&
        /arrow/ci/scripts/js_test.sh /arrow /build"

  #################################### C# #####################################

  ubuntu-csharp:
    # Usage:
    #   docker compose build ubuntu-csharp
    #   docker compose run ubuntu-csharp
    image: ${REPO}:${ARCH}-ubuntu-22.04-csharp-${DOTNET}
    build:
      context: .
      dockerfile: ci/docker/ubuntu-22.04-csharp.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-22.04-csharp-${DOTNET}
      args:
        arch: ${ARCH}
        dotnet: ${DOTNET}
        platform: jammy
    shm_size: *shm-size
    volumes: *ubuntu-volumes
    command: &csharp-command >
      /bin/bash -c "
        /arrow/ci/scripts/csharp_build.sh /arrow &&
        /arrow/ci/scripts/csharp_test.sh /arrow &&
        /arrow/ci/scripts/csharp_pack.sh /arrow"

  ############################## Integration ##################################

  conda-integration:
    # Usage:
    #   docker compose build conda-cpp
    #   docker compose build conda-integration
    #   docker compose run conda-integration
    image: ${REPO}:${ARCH}-conda-integration
    build:
      context: .
      dockerfile: ci/docker/conda-integration.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-integration
      args:
        repo: ${REPO}
        arch: ${ARCH}
        # Use a newer JDK as it seems to improve stability
        jdk: 17
        maven: ${MAVEN}
        node: ${NODE}
    volumes: *conda-volumes
    environment:
      <<: [*common, *ccache]
      ARCHERY_INTEGRATION_WITH_GO: 0
      ARCHERY_INTEGRATION_WITH_NANOARROW: 0
      ARCHERY_INTEGRATION_WITH_RUST: 0
      # Tell Archery where Arrow binaries are located
      ARROW_CPP_EXE_PATH: /build/cpp/debug
      ARROW_NANOARROW_PATH: /build/nanoarrow
      ARROW_RUST_EXE_PATH: /build/rust/debug
    command:
      ["git config --global --add safe.directory /arrow/go &&
        /arrow/ci/scripts/integration_arrow_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_arrow.sh /arrow /build"]

  ################################ Docs #######################################

  debian-docs:
    # Usage:
    #   docker compose build debian-cpp
    #   docker compose build debian-python
    #   docker compose build debian-docs
    #   docker compose run --rm debian-docs
    image: ${REPO}:${ARCH}-debian-${DEBIAN}-docs
    build:
      context: .
      dockerfile: ci/docker/linux-apt-docs.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-debian-${DEBIAN}-docs
      args:
        r: ${R}
        jdk: ${JDK}
        maven: ${MAVEN}
        node: ${NODE}
        base: ${REPO}:${ARCH}-debian-${DEBIAN}-python-3
    # This is for Chromium used by Mermaid. Chromium uses namespace
    # isolation for security by default.
    cap_add:
      - SYS_ADMIN
    environment:
      <<: [*common, *ccache]
      ARROW_CUDA: "ON"
      ARROW_CXX_FLAGS_DEBUG: "-g1"
      ARROW_C_FLAGS_DEBUG: "-g1"
      ARROW_HOME: "/tmp/local"
      ARROW_JAVA_SKIP_GIT_PLUGIN:
      ARROW_SUBSTRAIT: "ON"
      BUILD_DOCS_C_GLIB: "ON"
      BUILD_DOCS_CPP: "ON"
      BUILD_DOCS_JS: "ON"
      BUILD_DOCS_PYTHON: "ON"
      BUILD_DOCS_R: "ON"
    volumes: *debian-volumes
    command: >
      /bin/bash -c "
        sudo mkdir -p /build /ccache &&
        sudo chown -R `id --user --name`: /build /ccache &&
        /arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/c_glib_build.sh /arrow /build &&
        /arrow/ci/scripts/r_build.sh /arrow /build &&
        /arrow/ci/scripts/js_build.sh /arrow /build"

  ################################# Tools #####################################

  ubuntu-lint:
    # Usage:
    #   docker compose build ubuntu-cpp
    #   docker compose build ubuntu-lint
    #   docker compose run ubuntu-lint
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-lint
    build:
      context: .
      dockerfile: ci/docker/linux-apt-lint.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-lint
      args:
        base: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-cpp
        clang_tools: ${CLANG_TOOLS}
    environment:
      <<: [*common, *ccache]
    volumes: *ubuntu-volumes
    command: >
      /bin/bash -c "
        git config --global --add safe.directory /arrow &&
        python3 -m venv /build/pyvenv &&
        source /build/pyvenv/bin/activate &&
        pip install -U pip setuptools &&
        pip install arrow/dev/archery[lint] &&
        archery lint --all --no-clang-tidy --no-iwyu --no-numpydoc --src /arrow"

  ######################### Integration Tests #################################

  postgres:
    # required for the impala service
    image: postgres
    ports:
      - 5432:5432
    environment:
      POSTGRES_PASSWORD: postgres

  impala:
    # required for the hdfs tests
    image: ibisproject/impala:latest
    hostname: impala
    links:
      - postgres:postgres
    environment:
      PGPASSWORD: postgres
    ports:
      # HDFS
      - 9020:9020
      - 50070:50070
      - 50075:50075
      - 8020:8020
      - 8042:8042
      # Hive
      - 9083:9083
      # Impala
      - 21000:21000
      - 21050:21050
      - 25000:25000
      - 25010:25010
      - 25020:25020

  conda-python-hdfs:
    # Usage:
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-hdfs
    #   docker compose run conda-python-hdfs
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS}
    build:
      context: .
      dockerfile: ci/docker/conda-python-hdfs.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-hdfs-${HDFS}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        jdk: ${JDK}
        maven: ${MAVEN}
        hdfs: ${HDFS}
    links:
      - impala:impala
    environment:
      <<: [*common, *ccache]
      ARROW_ENGINE: "OFF"
      ARROW_FLIGHT: "OFF"
      ARROW_FLIGHT_SQL: "OFF"
      ARROW_HDFS: "ON"
      ARROW_HDFS_TEST_HOST: impala
      ARROW_HDFS_TEST_PORT: 8020
      ARROW_HDFS_TEST_USER: hdfs
      ARROW_S3: "OFF"
    shm_size: *shm-size
    volumes: &conda-maven-volumes
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}maven-cache:/root/.m2:delegated
      - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_hdfs.sh /arrow /build"]

  conda-python-spark:
    # Usage:
    #   docker compose build conda-cpp
    #   docker compose build conda-python
    #   docker compose build conda-python-spark
    #   docker compose run conda-python-spark
    image: ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK}
    build:
      context: .
      dockerfile: ci/docker/conda-python-spark.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-conda-python-${PYTHON}-spark-${SPARK}
      args:
        repo: ${REPO}
        arch: ${ARCH}
        python: ${PYTHON}
        jdk: ${JDK}
        maven: ${MAVEN}
        spark: ${SPARK}
        numpy: ${NUMPY}
    shm_size: *shm-size
    environment:
      <<: [*common, *ccache]
    volumes: *conda-maven-volumes
    command:
      ["/arrow/ci/scripts/cpp_build.sh /arrow /build &&
        /arrow/ci/scripts/python_build.sh /arrow /build &&
        /arrow/ci/scripts/integration_spark.sh /arrow /spark"]

  ################################# Source Verification #####################################

  conda-verify-rc:
    image: ubuntu:${UBUNTU}
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}conda-ccache:/ccache:delegated
    shm_size: '1gb'
    environment:
      <<: [*common, *ccache]
      CMAKE_GENERATOR: Ninja
      DEBIAN_FRONTEND: "noninteractive"
      DOTNET_SYSTEM_GLOBALIZATION_INVARIANT: 1
      TEST_APT: 0  # would require docker-in-docker
      TEST_YUM: 0
      USE_CONDA: 1
    command: >
      /bin/bash -c "
        apt update -y && apt install -y curl git gnupg tzdata wget &&
        git config --global --add safe.directory /arrow &&
        /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}"

  almalinux-verify-rc:
    # Usage:
    #   docker compose build almalinux-verify-rc
    #   docker compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 almalinux-verify-rc
    # Parameters:
    #   ALMALINUX: 8
    image: ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc
    build:
      context: .
      dockerfile: ci/docker/almalinux-${ALMALINUX}-verify-rc.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-almalinux-${ALMALINUX}-verify-rc
      args:
        repo: ${REPO}
        arch: ${ARCH}
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}almalinux-ccache:/ccache:delegated
    shm_size: '1gb'
    environment:
      <<: [*common, *ccache]
      CMAKE_GENERATOR: Ninja
      TEST_APT: 0  # would require docker-in-docker
      TEST_YUM: 0
    command: >
      /bin/bash -c "
        /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}"

  ubuntu-verify-rc:
    # Usage:
    #   docker compose build ubuntu-verify-rc
    #   docker compose run -e VERIFY_VERSION=6.0.1 -e VERIFY_RC=1 ubuntu-verify-rc
    # Parameters:
    #   UBUNTU: 20.04, 22.04
    image: ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc
    build:
      context: .
      dockerfile: ci/docker/ubuntu-${UBUNTU}-verify-rc.dockerfile
      cache_from:
        - ${REPO}:${ARCH}-ubuntu-${UBUNTU}-verify-rc
      args:
        repo: ${REPO}
        arch: ${ARCH}
    volumes:
      - .:/arrow:delegated
      - ${DOCKER_VOLUME_PREFIX}ubuntu-ccache:/ccache:delegated
    shm_size: '1gb'
    environment:
      <<: [*common, *ccache]
      CMAKE_GENERATOR: Ninja
      TEST_APT: 0  # would require docker-in-docker
      TEST_YUM: 0
    command: >
      /bin/bash -c "
        git config --global --add safe.directory /arrow &&
        /arrow/dev/release/verify-release-candidate.sh $${VERIFY_VERSION} $${VERIFY_RC}"

secrets:
  github_repository_owner:
    environment: GITHUB_REPOSITORY_OWNER
  github_token:
    environment: GITHUB_TOKEN
  vcpkg_binary_sources:
    environment: VCPKG_BINARY_SOURCES