Rushidave sync fork to upstream

.github/dependabot.yml

@@ -0,0 +1,7 @@
+version: 2
+updates:
+  # Maintain dependencies for GitHub Actions
+  - package-ecosystem: "github-actions"
+    directory: "/"
+    schedule:
+      interval: "daily"

.github/workflows/codeql-analysis.yml

@@ -0,0 +1,67 @@
+---
+# For most projects, this workflow file will not need changing; you simply need
+# to commit it to your repository.
+#
+# You may wish to alter this file to override the set of languages analyzed,
+# or to provide custom queries or build logic.
+#
+# ******** NOTE ********
+# We have attempted to detect the languages in your repository. Please check
+# the `language` matrix defined below to confirm you have the correct set of
+# supported CodeQL languages.
+#
+name: CodeQL
+on:
+  push:
+    branches: [master]
+  pull_request:
+    # The branches below must be a subset of the branches above
+    branches: [master]
+  schedule:
+    - cron: 19 10 * * 6
+jobs:
+  analyze:
+    name: Analyze
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+      security-events: write
+    strategy:
+      fail-fast: false
+      matrix:
+        language: [python]
+        # CodeQL supports [ 'cpp', 'csharp', 'go', 'java', 'javascript', 'python' ]
+        # Learn more:
+        # https://docs.github.com/en/free-pro-team@latest/github/finding-security-vulnerabilities-and-errors-in-your-code/configuring-code-scanning#changing-the-languages-that-are-analyzed
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+    # Initializes the CodeQL tools for scanning.
+      - name: Initialize CodeQL
+        uses: github/codeql-action/init@v3
+        with:
+          languages: ${{ matrix.language }}
+        # If you wish to specify custom queries, you can do so here or in a config file.
+        # By default, queries listed here will override any specified in a config file.
+        # Prefix the list here with "+" to use these queries and those in the config file.
+        # queries: ./path/to/local/query, your-org/your-repo/queries@main
+
+    # Autobuild attempts to build any compiled languages  (C/C++, C#, or Java).
+    # If this step fails, then you should remove it and run the build manually (see below)
+      - name: Autobuild
+        uses: github/codeql-action/autobuild@v3
+
+    # ℹ️ Command-line programs to run using the OS shell.
+    # 📚 https://git.io/JvXDl
+
+    # ✏️ If the Autobuild fails above, remove it and uncomment the following three lines
+    #    and modify them (or add more) to build your code if your project
+    #    uses a compiled language
+
+    #- run: |
+    #   make bootstrap
+    #   make release
+      - name: Perform CodeQL Analysis
+        uses: github/codeql-action/analyze@v3

.github/workflows/python-package.yml

@@ -0,0 +1,198 @@
+name: CI/CD
+
+on:
+  push:
+    branches: ["master"]
+  pull_request:
+    branches: ["master"]
+  release:
+    types: [created]
+    branches:
+      - 'master'
+  workflow_dispatch:
+
+env:
+  FORCE_COLOR: "1"  # Make tools pretty.
+  PIP_DISABLE_PIP_VERSION_CHECK: "1"
+  PIP_NO_PYTHON_VERSION_WARNING: "1"
+  PYTHON_LATEST: "3.12"
+  KAFKA_LATEST: "2.6.0"
+
+  # For re-actors/checkout-python-sdist
+  sdist-artifact: python-package-distributions
+
+jobs:
+
+  build-sdist:
+    name: 📦 Build the source distribution
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout project
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_LATEST }}
+          cache: pip
+      - run: python -m pip install build
+        name: Install core libraries for build and install
+      - name: Build artifacts
+        run: python -m build
+      - name: Upload built artifacts for testing
+        uses: actions/upload-artifact@v3
+        with:
+          name: ${{ env.sdist-artifact }}
+          # NOTE: Exact expected file names are specified here
+          # NOTE: as a safety measure — if anything weird ends
+          # NOTE: up being in this dir or not all dists will be
+          # NOTE: produced, this will fail the workflow.
+          path: dist/${{ env.sdist-name }}
+          retention-days: 15
+
+  test-python:
+    name: Tests on ${{ matrix.python-version }}
+    needs:
+      - build-sdist
+    runs-on: ubuntu-latest
+    continue-on-error: ${{ matrix.experimental }}
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version:
+          - "3.8"
+          - "3.9"
+          - "3.10"
+          - "3.11"
+          - "3.12"
+        experimental: [ false ]
+        include:
+          - python-version: "pypy3.9"
+            experimental: true
+          - python-version: "~3.13.0-0"
+            experimental: true
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 11
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: pip
+          cache-dependency-path: |
+            requirements-dev.txt
+      - name: Check Java installation
+        run: source travis_java_install.sh
+      - name: Pull Kafka releases
+        run: ./build_integration.sh
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
+        # TODO: Cache releases to expedite testing
+      - name: Install dependencies
+        run: |
+          sudo apt install -y libsnappy-dev libzstd-dev
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh-actions
+          pip install .
+          pip install -r requirements-dev.txt
+      - name: Test with tox
+        run: tox
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ env.KAFKA_LATEST }}
+
+  test-kafka:
+    name: Tests for Kafka ${{ matrix.kafka-version }}
+    needs:
+      - build-sdist
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        kafka-version:
+          - "0.8.2.2"
+          - "0.9.0.1"
+          - "0.10.2.2"
+          - "0.11.0.2"
+          - "0.11.0.3"
+          - "1.1.1"
+          - "2.4.0"
+          - "2.5.0"
+          - "2.6.0"
+    steps:
+      - name: Checkout the source code
+        uses: actions/checkout@v4
+        with:
+          fetch-depth: 0
+      - name: Setup java
+        uses: actions/setup-java@v4
+        with:
+          distribution: temurin
+          java-version: 8
+      - name: Set up Python
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ env.PYTHON_LATEST }}
+          cache: pip
+          cache-dependency-path: |
+            requirements-dev.txt
+      - name: Pull Kafka releases
+        run: ./build_integration.sh
+        env:
+          # This is fast enough as long as you pull only one release at a time,
+          # no need to worry about caching
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ matrix.kafka-version }}
+      - name: Install dependencies
+        run: |
+          sudo apt install -y libsnappy-dev libzstd-dev
+          python -m pip install --upgrade pip
+          python -m pip install tox tox-gh-actions
+          pip install .
+          pip install -r requirements-dev.txt
+      - name: Test with tox
+        run: tox
+        env:
+          PLATFORM: ${{ matrix.platform }}
+          KAFKA_VERSION: ${{ matrix.kafka-version }}
+
+  check:  # This job does nothing and is only used for the branch protection
+    name: ✅ Ensure the required checks passing
+    if: always()
+    needs:
+      - build-sdist
+      - test-python
+      - test-kafka
+    runs-on: ubuntu-latest
+    steps:
+    - name: Decide whether the needed jobs succeeded or failed
+      uses: re-actors/alls-green@release/v1
+      with:
+        jobs: ${{ toJSON(needs) }}
+  publish:
+    name: 📦 Publish to PyPI
+    runs-on: ubuntu-latest
+    needs: [build-sdist]
+    permissions:
+      id-token: write
+    environment: pypi
+    if: github.event_name == 'release' && github.event.action == 'created'
+    steps:
+      - name: Download the sdist artifact
+        uses: actions/download-artifact@v3
+        with:
+          name: artifact
+          path: dist
+      - name: Publish package to PyPI
+        uses: pypa/gh-action-pypi-publish@release/v1
+        with:
+          password: ${{ secrets.PYPI_API_TOKEN }}

Makefile

@@ -20,14 +20,14 @@ test37: build-integration
 test27: build-integration
 	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) tox -e py27 -- $(FLAGS)
 
-# Test using py.test directly if you want to use local python. Useful for other
+# Test using pytest directly if you want to use local python. Useful for other
 # platforms that require manual installation for C libraries, ie. Windows.
 test-local: build-integration
-	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) py.test \
+	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \
 		--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF $(FLAGS) kafka test
 
 cov-local: build-integration
-	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) py.test \
+	KAFKA_VERSION=$(KAFKA_VERSION) SCALA_VERSION=$(SCALA_VERSION) pytest \
 		--pylint --pylint-rcfile=pylint.rc --pylint-error-types=EF --cov=kafka \
 		--cov-config=.covrc --cov-report html $(FLAGS) kafka test
 	@echo "open file://`pwd`/htmlcov/index.html"

README.rst

@@ -7,10 +7,16 @@ Kafka Python client
     :target: https://pypi.python.org/pypi/kafka-python
 .. image:: https://coveralls.io/repos/dpkp/kafka-python/badge.svg?branch=master&service=github
     :target: https://coveralls.io/github/dpkp/kafka-python?branch=master
-.. image:: https://travis-ci.org/dpkp/kafka-python.svg?branch=master
-    :target: https://travis-ci.org/dpkp/kafka-python
 .. image:: https://img.shields.io/badge/license-Apache%202-blue.svg
     :target: https://github.com/dpkp/kafka-python/blob/master/LICENSE
+.. image:: https://img.shields.io/pypi/dw/kafka-python.svg
+    :target: https://pypistats.org/packages/kafka-python
+.. image:: https://img.shields.io/pypi/v/kafka-python.svg
+    :target: https://pypi.org/project/kafka-python
+.. image:: https://img.shields.io/pypi/implementation/kafka-python
+    :target: https://github.com/dpkp/kafka-python/blob/master/setup.py
+
+
 
 Python client for the Apache Kafka distributed stream processing system.
 kafka-python is designed to function much like the official java client, with a

docs/usage.rst

@@ -8,6 +8,8 @@ KafkaConsumer
 .. code:: python
 
     from kafka import KafkaConsumer
+    import json
+    import msgpack
 
     # To consume latest messages and auto-commit offsets
     consumer = KafkaConsumer('my-topic',
@@ -57,6 +59,8 @@ KafkaProducer
 
     from kafka import KafkaProducer
     from kafka.errors import KafkaError
+    import msgpack
+    import json
 
     producer = KafkaProducer(bootstrap_servers=['broker1:1234'])
 
@@ -108,3 +112,52 @@ KafkaProducer
 
     # configure multiple retries
     producer = KafkaProducer(retries=5)
+
+
+ClusterMetadata
+=============
+.. code:: python
+
+    from kafka.cluster import ClusterMetadata
+
+    clusterMetadata = ClusterMetadata(bootstrap_servers=['broker1:1234'])
+
+    # get all brokers metadata
+    print(clusterMetadata.brokers())
+
+    # get specific broker metadata
+    print(clusterMetadata.broker_metadata('bootstrap-0'))
+
+    # get all partitions of a topic
+    print(clusterMetadata.partitions_for_topic("topic"))
+
+    # list topics 
+    print(clusterMetadata.topics())
+
+
+KafkaAdminClient
+=============
+.. code:: python
+    from kafka import KafkaAdminClient
+    from kafka.admin import NewTopic
+
+    admin = KafkaAdminClient(bootstrap_servers=['broker1:1234'])
+
+    # create a new topic
+    topics_list = []
+    topics_list.append(NewTopic(name="testtopic", num_partitions=1, replication_factor=1))
+    admin.create_topics(topics_list,timeout_ms=None, validate_only=False)
+
+    # delete a topic
+    admin.delete_topics(['testtopic'])
+
+    # list consumer groups
+    print(admin.list_consumer_groups())
+
+    # get consumer group details
+    print(admin.describe_consumer_groups('cft-plt-qa.connect'))
+
+    # get consumer group offset
+    print(admin.list_consumer_group_offsets('cft-plt-qa.connect'))
+
+