Merge branch 'main' into ibis-project#8989

Author: kaijennissen

.devcontainer/Dockerfile

@@ -0,0 +1,4 @@
+FROM mcr.microsoft.com/vscode/devcontainers/python:3.11
+
+RUN apt-get update && \
+  apt-get install -y --no-install-recommends libgdal-dev

.devcontainer/devcontainer.json

@@ -1,6 +1,6 @@
 {
-  "image": "mcr.microsoft.com/vscode/devcontainers/python:3.11",
-  "updateContentCommand": ".devcontainer/updateContent.sh",
+  "build": { "dockerfile": "Dockerfile" },
+  "postCreateCommand": ".devcontainer/postCreate.sh",
   "customizations": {
     "codespaces": {
       "openFiles": ["docs/tutorials/getting_started.qmd"]

.devcontainer/postCreate.sh

@@ -0,0 +1,5 @@
+#!/bin/sh
+
+# install ibis
+python3 -m pip install ipython
+POETRY_DYNAMIC_VERSIONING=false python3 -m pip install -e '.[duckdb,clickhouse,examples,geospatial]'

.devcontainer/updateContent.sh

@@ -35,11 +35,16 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       - run: echo "No build required"
+  test_pyspark:
+    runs-on: ubuntu-latest
+    steps:
+      - run: echo "No build required"
   backends:
     # this job exists so that we can use a single job from this workflow to gate merging
     runs-on: ubuntu-latest
     needs:
       - test_backends_min_version
       - test_backends
+      - test_pyspark
     steps:
       - run: exit 0

.github/workflows/ibis-backends-skip-helper.yml

@@ -135,7 +135,7 @@ This allows you to combine the flexibility of Python with the scale and performa
 
 Ibis supports 20+ backends:
 
-- [Apache Arrow DataFusion](https://ibis-project.org/backends/datafusion/)
+- [Apache DataFusion](https://ibis-project.org/backends/datafusion/)
 - [Apache Druid](https://ibis-project.org/backends/druid/)
 - [Apache Flink](https://ibis-project.org/backends/flink)
 - [Apache Impala](https://ibis-project.org/backends/impala/)

README.md

@@ -94,7 +94,7 @@ services:
       - trino
 
   minio:
-    image: bitnami/minio:2024.4.6
+    image: bitnami/minio:2024.4.18
     environment:
       MINIO_ROOT_USER: accesskey
       MINIO_ROOT_PASSWORD: secretkey

compose.yaml

@@ -6,6 +6,7 @@ dependencies:
   - python =3.10
   - atpublic >=2.3
   - bidict >=0.22.1
+  - black >=22.1.0,<25
   - clickhouse-connect >=0.5.23
   - dask >=2022.9.1
   - datafusion >=0.6

conda/environment-arm64-flink.yml

@@ -6,6 +6,7 @@ dependencies:
   - python >=3.10
   - atpublic >=2.3
   - bidict >=0.22.1
+  - black >=22.1.0,<25
   - clickhouse-connect >=0.5.23
   - dask >=2022.9.1
   - datafusion >=0.6

conda/environment-arm64.yml

@@ -6,6 +6,7 @@ dependencies:
   - apache-flink
   - atpublic >=2.3
   - bidict >=0.22.1
+  - black >=22.1.0,<25
   - clickhouse-connect >=0.5.23
   - dask >=2022.9.1
   - datafusion >=0.6

conda/environment.yml

@@ -97,7 +97,7 @@ for manager, params in managers.items():
     )
     print()
     print("    ```sh")
-    print("    pip install -e '.[all]'")
+    print("    pip install -e .")
     print("    ```")
     print()
 ```
@@ -200,7 +200,7 @@ For a better development experience see the `conda/mamba` or `nix` setup instruc
 1. Install ibis in development mode
 
    ```sh
-   pip install -e '.[all]'
+   pip install -e .
    ```
 
 :::