object_store.yml

# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements.  See the NOTICE file
# distributed with this work for additional information
# regarding copyright ownership.  The ASF licenses this file
# to you under the Apache License, Version 2.0 (the
# "License"); you may not use this file except in compliance
# with the License.  You may obtain a copy of the License at
#
#   http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing,
# software distributed under the License is distributed on an
# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
# KIND, either express or implied.  See the License for the
# specific language governing permissions and limitations
# under the License.

---
# tests for `object_store` crate
name: object_store

concurrency:
  group: ${{ github.repository }}-${{ github.head_ref || github.sha }}-${{ github.workflow }}
  cancel-in-progress: true

# trigger for all PRs that touch certain files and changes to master
on:
  push:
    branches:
      - master
  pull_request:
    paths:
      - object_store/**
      - .github/**

jobs:
  clippy:
    name: Clippy
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    defaults:
      run:
        working-directory: object_store
    steps:
      - uses: actions/checkout@v4
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
      - name: Setup Clippy
        run: rustup component add clippy
      # Run different tests for the library on its own as well as
      # all targets to ensure that it still works in the absence of
      # features that might be enabled by dev-dependencies of other
      # targets.
      - name: Run clippy with default features
        run: cargo clippy -- -D warnings
      - name: Run clippy with aws feature
        run: cargo clippy --features aws -- -D warnings
      - name: Run clippy with gcp feature
        run: cargo clippy --features gcp -- -D warnings
      - name: Run clippy with azure feature
        run: cargo clippy --features azure -- -D warnings
      - name: Run clippy with http feature
        run: cargo clippy --features http -- -D warnings
      - name: Run clippy with all features
        run: cargo clippy --all-features -- -D warnings
      - name: Run clippy with all features and all targets
        run: cargo clippy --all-features --all-targets -- -D warnings

  # test doc links still work
  #
  # Note that since object_store is not part of the main workspace,
  # this needs a separate docs job as it is not covered by
  # `cargo doc --workspace`
  docs:
    name: Rustdocs
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: object_store
    env:
      RUSTDOCFLAGS: "-Dwarnings"
    steps:
      - uses: actions/checkout@v4
      - name: Run cargo doc
        run: cargo doc --document-private-items --no-deps --workspace --all-features

  # test the crate
  # This runs outside a container to workaround lack of support for passing arguments
  # to service containers - https://github.com/orgs/community/discussions/26688
  linux-test:
    name: Emulator Tests
    runs-on: ubuntu-latest
    defaults:
      run:
        working-directory: object_store
    env:
      # Disable full debug symbol generation to speed up CI build and keep memory down
      # "1" means line tables only, which is useful for panic tracebacks.
      RUSTFLAGS: "-C debuginfo=1"
      RUST_BACKTRACE: "1"
      # Run integration tests
      TEST_INTEGRATION: 1
      EC2_METADATA_ENDPOINT: http://localhost:1338
      AZURE_CONTAINER_NAME: test-bucket
      AZURE_STORAGE_USE_EMULATOR: "1"
      AZURITE_BLOB_STORAGE_URL: "http://localhost:10000"
      AZURITE_QUEUE_STORAGE_URL: "http://localhost:10001"
      AWS_BUCKET: test-bucket
      AWS_DEFAULT_REGION: "us-east-1"
      AWS_ACCESS_KEY_ID: test
      AWS_SECRET_ACCESS_KEY: test
      AWS_ENDPOINT: http://localhost:4566
      AWS_ALLOW_HTTP: true
      AWS_COPY_IF_NOT_EXISTS: dynamo:test-table:2000
      AWS_CONDITIONAL_PUT: dynamo:test-table:2000
      AWS_SERVER_SIDE_ENCRYPTION: aws:kms
      HTTP_URL: "http://localhost:8080"
      GOOGLE_BUCKET: test-bucket
      GOOGLE_SERVICE_ACCOUNT: "/tmp/gcs.json"

    steps:
      - uses: actions/checkout@v4

      # We are forced to use docker commands instead of service containers as we need to override the entrypoints
      # which is currently not supported - https://github.com/actions/runner/discussions/1872
      - name: Configure Fake GCS Server (GCP emulation)
        # Custom image - see fsouza/fake-gcs-server#1164
        run: |
          echo "GCS_CONTAINER=$(docker run -d -p 4443:4443 tustvold/fake-gcs-server -scheme http -backend memory -public-host localhost:4443)" >> $GITHUB_ENV
          # Give the container a moment to start up prior to configuring it
          sleep 1
          curl -v -X POST --data-binary '{"name":"test-bucket"}' -H "Content-Type: application/json" "http://localhost:4443/storage/v1/b"
          echo '{"gcs_base_url": "http://localhost:4443", "disable_oauth": true, "client_email": "", "private_key": "", "private_key_id": ""}' > "$GOOGLE_SERVICE_ACCOUNT"

      - name: Setup WebDav
        run: docker run -d -p 8080:80 rclone/rclone serve webdav /data --addr :80

      - name: Setup LocalStack (AWS emulation)
        run: |
          echo "LOCALSTACK_CONTAINER=$(docker run -d -p 4566:4566 localstack/localstack:3.8.1)" >> $GITHUB_ENV
          echo "EC2_METADATA_CONTAINER=$(docker run -d -p 1338:1338 amazon/amazon-ec2-metadata-mock:v1.9.2 --imdsv2)" >> $GITHUB_ENV
          aws --endpoint-url=http://localhost:4566 s3 mb s3://test-bucket
          aws --endpoint-url=http://localhost:4566 s3api create-bucket --bucket test-object-lock --object-lock-enabled-for-bucket 
          aws --endpoint-url=http://localhost:4566 dynamodb create-table --table-name test-table --key-schema AttributeName=path,KeyType=HASH AttributeName=etag,KeyType=RANGE --attribute-definitions AttributeName=path,AttributeType=S AttributeName=etag,AttributeType=S --provisioned-throughput ReadCapacityUnits=5,WriteCapacityUnits=5

          KMS_KEY=$(aws --endpoint-url=http://localhost:4566 kms create-key --description "test key")
          echo "AWS_SSE_KMS_KEY_ID=$(echo $KMS_KEY | jq -r .KeyMetadata.KeyId)" >> $GITHUB_ENV

      - name: Configure Azurite (Azure emulation)
        # the magical connection string is from
        # https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azurite?tabs=visual-studio#http-connection-strings
        run: |
          echo "AZURITE_CONTAINER=$(docker run -d -p 10000:10000 -p 10001:10001 -p 10002:10002 mcr.microsoft.com/azure-storage/azurite)" >> $GITHUB_ENV
          az storage container create -n test-bucket --connection-string 'DefaultEndpointsProtocol=http;AccountName=devstoreaccount1;AccountKey=Eby8vdM02xNOcqFlqUwJPLlmEtlCDXJ1OUzFT50uSRZ6IFsuFq2UVErCz4I6tq/K1SZFPTOtr/KBHBeksoGMGw==;BlobEndpoint=http://localhost:10000/devstoreaccount1;QueueEndpoint=http://localhost:10001/devstoreaccount1;'

      - name: Setup Rust toolchain
        run: |
          rustup toolchain install stable
          rustup default stable

      - name: Run object_store tests
        run: cargo test --features=aws,azure,gcp,http

      - name: Run object_store tests (AWS native conditional put)
        run: cargo test --features=aws
        env:
          AWS_CONDITIONAL_PUT: etag-put-if-not-exists
          AWS_COPY_IF_NOT_EXISTS: multipart

      - name: GCS Output
        if: ${{ !cancelled() }}
        run: docker logs $GCS_CONTAINER

      - name: LocalStack Output
        if: ${{ !cancelled() }}
        run: docker logs $LOCALSTACK_CONTAINER

      - name: EC2 Metadata Output
        if: ${{ !cancelled() }}
        run: docker logs $EC2_METADATA_CONTAINER

      - name: Azurite Output
        if: ${{ !cancelled() }}
        run: docker logs $AZURITE_CONTAINER

  # test the object_store crate builds against wasm32 in stable rust
  wasm32-build:
    name: Build wasm32
    runs-on: ubuntu-latest
    container:
      image: amd64/rust
    defaults:
      run:
        working-directory: object_store
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Setup Rust toolchain
        uses: ./.github/actions/setup-builder
        with:
          target: wasm32-unknown-unknown,wasm32-wasi
      - name: Build wasm32-unknown-unknown
        run: cargo build --target wasm32-unknown-unknown
      - name: Build wasm32-wasi
        run: cargo build --target wasm32-wasi

  windows:
    name: cargo test LocalFileSystem (win64)
    runs-on: windows-latest
    defaults:
      run:
        working-directory: object_store
    steps:
      - uses: actions/checkout@v4
        with:
          submodules: true
      - name: Run LocalFileSystem tests
        run: cargo test local::tests