{"id":13759550,"url":"https://github.com/deviceinsight/kafkactl","last_synced_at":"2026-10-19T17:11:29.610Z","repository":{"id":37951289,"uuid":"161144285","full_name":"deviceinsight/kafkactl","owner":"deviceinsight","description":"Command Line Tool for managing Apache Kafka","archived":false,"fork":false,"pushed_at":"2026-10-09T06:06:59.000Z","size":2384,"stargazers_count":1072,"open_issues_count":12,"forks_count":113,"subscribers_count":16,"default_branch":"main","last_synced_at":"2026-10-18T14:16:25.080Z","etag":null,"topics":["apache-kafka","avro","cli","fish","golang","kafka","kubernetes","protobuf","zsh"],"latest_commit_sha":null,"homepage":"https://deviceinsight.github.io/kafkactl/","language":"Go","has_issues":true,"has_wiki":null,"has_pages":null,"mirror_url":null,"source_name":null,"license":"apache-2.0","status":null,"scm":"git","pull_requests_enabled":true,"icon_url":"https://github.com/deviceinsight.png","metadata":{"files":{"readme":"README.adoc","changelog":"CHANGELOG.md","contributing":".github/contributing.md","funding":null,"license":"LICENSE.md","code_of_conduct":null,"threat_model":null,"audit":null,"citation":null,"codeowners":null,"security":null,"support":null,"governance":null,"roadmap":null,"authors":null,"dei":null,"publiccode":null,"codemeta":null,"zenodo":null,"notice":null,"maintainers":null,"copyright":null,"agents":null,"dco":null,"cla":null}},"created_at":"2018-12-10T08:43:54.000Z","updated_at":"2026-10-13T13:18:13.000Z","dependencies_parsed_at":"2026-01-02T18:12:12.631Z","dependency_job_id":null,"html_url":"https://github.com/deviceinsight/kafkactl","commit_stats":{"total_commits":487,"total_committers":48,"mean_commits":"10.145833333333334","dds":0.2915811088295688,"last_synced_commit":"70ab0e03a9d300d3fafdbb60b4e80d118aa03046"},"previous_names":[],"tags_count":97,"template":false,"template_full_name":null,"purl":"pkg:github/deviceinsight/kafkactl","repository_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deviceinsight%2Fkafkactl","tags_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deviceinsight%2Fkafkactl/tags","releases_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deviceinsight%2Fkafkactl/releases","manifests_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deviceinsight%2Fkafkactl/manifests","owner_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners/deviceinsight","download_url":"https://codeload.github.com/deviceinsight/kafkactl/tar.gz/refs/heads/main","sbom_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories/deviceinsight%2Fkafkactl/sbom","scorecard":{"id":337575,"data":{"date":"2025-08-11","repo":{"name":"github.com/deviceinsight/kafkactl","commit":"b35f40e7d91ccf1738f731c91d7e2ab1d8743097"},"scorecard":{"version":"v5.2.1-40-gf6ed084d","commit":"f6ed084d17c9236477efd66e5b258b9d4cc7b389"},"score":4.9,"checks":[{"name":"Dangerous-Workflow","score":10,"reason":"no dangerous workflow patterns detected","details":null,"documentation":{"short":"Determines if the project's GitHub Action workflows avoid dangerous patterns.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#dangerous-workflow"}},{"name":"Token-Permissions","score":0,"reason":"detected GitHub workflow tokens with excessive permissions","details":["Warn: no topLevel permission defined: .github/workflows/lint_test.yml:1","Warn: no topLevel permission defined: .github/workflows/publish_ghpages.yml:1","Warn: no topLevel permission defined: .github/workflows/release.yml:1","Info: no jobLevel write permissions found"],"documentation":{"short":"Determines if the project's workflows follow the principle of least privilege.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#token-permissions"}},{"name":"Code-Review","score":2,"reason":"Found 4/17 approved changesets -- score normalized to 2","details":null,"documentation":{"short":"Determines if the project requires human code review before pull requests (aka merge requests) are merged.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#code-review"}},{"name":"Maintained","score":10,"reason":"30 commit(s) and 12 issue activity found in the last 90 days -- score normalized to 10","details":null,"documentation":{"short":"Determines if the project is \"actively maintained\".","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#maintained"}},{"name":"Binary-Artifacts","score":10,"reason":"no binaries found in the repo","details":null,"documentation":{"short":"Determines if the project has generated executable (binary) artifacts in the source repository.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#binary-artifacts"}},{"name":"CII-Best-Practices","score":0,"reason":"no effort to earn an OpenSSF best practices badge detected","details":null,"documentation":{"short":"Determines if the project has an OpenSSF (formerly CII) Best Practices Badge.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#cii-best-practices"}},{"name":"Security-Policy","score":0,"reason":"security policy file not detected","details":["Warn: no security policy file detected","Warn: no security file to analyze","Warn: no security file to analyze","Warn: no security file to analyze"],"documentation":{"short":"Determines if the project has published a security policy.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#security-policy"}},{"name":"License","score":10,"reason":"license file detected","details":["Info: project has a license file: LICENSE.md:0","Info: FSF or OSI recognized license: Apache License 2.0: LICENSE.md:0"],"documentation":{"short":"Determines if the project has defined a license.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#license"}},{"name":"Fuzzing","score":0,"reason":"project is not fuzzed","details":["Warn: no fuzzer integrations found"],"documentation":{"short":"Determines if the project uses fuzzing.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#fuzzing"}},{"name":"Pinned-Dependencies","score":0,"reason":"dependency not pinned by hash detected -- score normalized to 0","details":["Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:16: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:17: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:27: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:28: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:34: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:59: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:60: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/lint_test.yml:64: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/lint_test.yml:71: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/lint_test.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/publish_ghpages.yml:13: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/publish_ghpages.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/publish_ghpages.yml:17: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/publish_ghpages.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/publish_ghpages.yml:25: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/publish_ghpages.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/release.yml:14: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/release.yml/main?enable=pin","Warn: GitHub-owned GitHubAction not pinned by hash: .github/workflows/release.yml:20: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/release.yml/main?enable=pin","Warn: third-party GitHubAction not pinned by hash: .github/workflows/release.yml:28: update your workflow using https://app.stepsecurity.io/secureworkflow/deviceinsight/kafkactl/release.yml/main?enable=pin","Warn: containerImage not pinned by hash: Dockerfile.Scratch:1","Warn: containerImage not pinned by hash: Dockerfile.Ubuntu:1: pin your Docker image by updating ubuntu:latest to ubuntu:latest@sha256:7c06e91f61fa88c08cc74f7e1b7c69ae24910d745357e0dfe1d2c0322aaf20f9","Info:   0 out of  12 GitHub-owned GitHubAction dependencies pinned","Info:   0 out of   3 third-party GitHubAction dependencies pinned","Info:   0 out of   2 containerImage dependencies pinned"],"documentation":{"short":"Determines if the project has declared and pinned the dependencies of its build process.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#pinned-dependencies"}},{"name":"Branch-Protection","score":-1,"reason":"internal error: error during branchesHandler.setup: internal error: githubv4.Query: Resource not accessible by integration","details":null,"documentation":{"short":"Determines if the default and release branches are protected with GitHub's branch protection settings.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#branch-protection"}},{"name":"Packaging","score":10,"reason":"packaging workflow detected","details":["Info: Project packages its releases by way of GitHub Actions.: .github/workflows/release.yml:9"],"documentation":{"short":"Determines if the project is published as a package that others can easily download, install, easily update, and uninstall.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#packaging"}},{"name":"Signed-Releases","score":0,"reason":"Project has not signed or included provenance with any releases.","details":["Warn: release artifact v5.11.1 not signed: https://api.github.com/repos/deviceinsight/kafkactl/releases/235647880","Warn: release artifact v5.11.0 not signed: https://api.github.com/repos/deviceinsight/kafkactl/releases/232809492","Warn: release artifact v5.10.1 not signed: https://api.github.com/repos/deviceinsight/kafkactl/releases/229403069","Warn: release artifact v5.10.0 not signed: https://api.github.com/repos/deviceinsight/kafkactl/releases/227326803","Warn: release artifact v5.9.0 not signed: https://api.github.com/repos/deviceinsight/kafkactl/releases/225485729","Warn: release artifact v5.11.1 does not have provenance: https://api.github.com/repos/deviceinsight/kafkactl/releases/235647880","Warn: release artifact v5.11.0 does not have provenance: https://api.github.com/repos/deviceinsight/kafkactl/releases/232809492","Warn: release artifact v5.10.1 does not have provenance: https://api.github.com/repos/deviceinsight/kafkactl/releases/229403069","Warn: release artifact v5.10.0 does not have provenance: https://api.github.com/repos/deviceinsight/kafkactl/releases/227326803","Warn: release artifact v5.9.0 does not have provenance: https://api.github.com/repos/deviceinsight/kafkactl/releases/225485729"],"documentation":{"short":"Determines if the project cryptographically signs release artifacts.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#signed-releases"}},{"name":"SAST","score":0,"reason":"SAST tool is not run on all commits -- score normalized to 0","details":["Warn: 0 commits out of 20 are checked with a SAST tool"],"documentation":{"short":"Determines if the project uses static code analysis.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#sast"}},{"name":"Vulnerabilities","score":10,"reason":"0 existing vulnerabilities detected","details":null,"documentation":{"short":"Determines if the project has open, known unfixed vulnerabilities.","url":"https://github.com/ossf/scorecard/blob/f6ed084d17c9236477efd66e5b258b9d4cc7b389/docs/checks.md#vulnerabilities"}}]},"last_synced_at":"2025-08-18T05:06:40.641Z","repository_id":37951289,"created_at":"2025-08-18T05:06:40.641Z","updated_at":"2025-08-18T05:06:40.641Z"},"host":{"name":"GitHub","url":"https://github.com","kind":"github","repositories_count":344872684,"owners_count":38362634,"icon_url":"https://github.com/github.png","version":null,"created_at":"2022-05-30T11:31:42.601Z","updated_at":"2026-10-03T21:59:58.778Z","status":"online","status_checked_at":"2026-10-19T02:00:06.414Z","response_time":53,"last_error":null,"robots_txt_status":"success","robots_txt_updated_at":"2025-07-24T06:49:26.215Z","robots_txt_url":"https://github.com/robots.txt","online":true,"can_crawl_api":true,"host_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub","repositories_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repositories","repository_names_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/repository_names","owners_url":"https://repos.ecosyste.ms/api/v1/hosts/GitHub/owners"}},"keywords":["apache-kafka","avro","cli","fish","golang","kafka","kubernetes","protobuf","zsh"],"created_at":"2024-08-03T13:00:55.257Z","updated_at":"2026-10-19T17:11:29.604Z","avatar_url":"https://github.com/deviceinsight.png","language":"Go","funding_links":[],"categories":["Uncategorized","Go","CLI Tools"],"sub_categories":["Uncategorized","Interactive Tools"],"readme":":toc:\n:toclevels: 2\n\n= kafkactl\n\nA command-line interface for interaction with Apache Kafka\n\nimage:https://github.com/deviceinsight/kafkactl/actions/workflows/lint_test.yml/badge.svg[Build Status,link=https://github.com/deviceinsight/kafkactl/actions]\n| image:https://img.shields.io/badge/command-docs-blue.svg[command docs,link=https://deviceinsight.github.io/kafkactl/]\n\n== Features\n\n* command auto-completion for bash, zsh, fish shell including dynamic completion for e.g. topics or consumer groups.\n* support for avro schemas\n* support for JSON schema registry\n* Configuration of different contexts\n* directly access kafka clusters inside your kubernetes cluster\n* support for consuming and producing protobuf-encoded messages\n\nimage::https://asciinema.org/a/vmxrTA0h8CAXPnJnSFk5uHKzr.svg[asciicast,link=https://asciinema.org/a/vmxrTA0h8CAXPnJnSFk5uHKzr]\n\n== Installation\n\nYou can install the pre-compiled binary or compile from source.\n\n=== Install the pre-compiled binary\n\n*homebrew*:\n\n[,bash]\n----\n# install kafkactl\nbrew install kafkactl\n# upgrade kafkactl\nbrew upgrade kafkactl\n----\n\n*winget*:\n[,bash]\n----\nwinget install kafkactl\n----\n\n*deb/rpm*:\n\nDownload the .deb or .rpm from the https://github.com/deviceinsight/kafkactl/releases[releases page] and install with dpkg -i and rpm -i respectively.\n\n*yay (AUR)*\n\nThere's a kafkactl https://aur.archlinux.org/packages/kafkactl/[AUR package] available for Arch. Install it with your AUR helper of choice (e.g. https://github.com/Jguer/yay[yay]):\n\n[,bash]\n----\nyay -S kafkactl\n----\n\n*manually*:\n\nDownload the pre-compiled binaries from the https://github.com/deviceinsight/kafkactl/releases[releases page] and copy to the desired location.\n\n=== Compiling from source\n\n[,bash]\n----\ngo install github.com/deviceinsight/kafkactl/v5@latest\n----\n\n*NOTE:* make sure that `kafkactl` is on PATH otherwise auto-completion won't work.\n\n== Configuration\n\nIf no config file is found, a default config is generated in `$HOME/.config/kafkactl/config.yml`.\nThis configuration is suitable to get started with a single node cluster on a local machine.\n\n=== Create a config file\n\nCreate `$HOME/.config/kafkactl/config.yml` with a definition of contexts that should be available\n\n[,yaml]\n----\ncontexts:\n  default:\n    brokers:\n      - localhost:9092\n  remote-cluster:\n    brokers:\n      - remote-cluster001:9092\n      - remote-cluster002:9092\n      - remote-cluster003:9092\n\n    # optional: tls config\n    tls:\n      enabled: true\n      ca: my-ca\n      cert: my-cert\n      certKey: my-key\n      # optional: passphrase for encrypted cert key (if omitted and key is encrypted, will prompt or use keyring)\n      certKeyPassphrase: my-passphrase\n      # set insecure to true to ignore all tls verification (defaults to false)\n      insecure: false\n\n    # optional: sasl support\n    sasl:\n      enabled: true\n      username: admin\n      # optional: if omitted, password is looked up in the OS keyring or prompted interactively\n      password: admin\n      # optional configure sasl mechanism as plaintext, scram-sha256, scram-sha512, oauth (defaults to plaintext)\n      mechanism: oauth\n      # optional configure sasl version as v0, v1 (defaults to not configured), Refer to: https://github.com/IBM/sarama/issues/3000#issuecomment-2415829478\n      version: v0\n      # optional tokenProvider configuration (only used for 'sasl.mechanism=oauth')\n      tokenprovider:\n        # plugin to use as token provider implementation (see plugin section)\n        plugin: azure\n        # optional: additional options passed to the plugin\n        options:\n          key: value\n\n    # optional: access clusters running kubernetes\n    kubernetes:\n      enabled: false\n      binary: kubectl #optional\n      kubeConfig: ~/.kube/config #optional\n      kubeContext: my-cluster\n      namespace: my-namespace\n      # optional: docker image to use (the tag of the image will be suffixed by `-scratch` or `-ubuntu` depending on command)\n      image: private.registry.com/deviceinsight/kafkactl\n      # optional: secret for private docker registry\n      imagePullSecret: registry-secret\n      # optional: secret containing tls certificates (e.g. ca.crt, cert.crt, key.key)\n      tlsSecret: tls-secret\n      # optional: inject SASL credentials into the pod via a Kubernetes secret instead of plaintext env vars.\n      # name and create are mutually exclusive.\n      saslSecret:\n        # reference an existing secret with keys \"username\" and \"password\" (kafkactl will not manage its lifecycle)\n        name: sasl-secret\n        # alternatively, let kafkactl create a short-lived secret from the local sasl config and delete it after the pod exits\n        create: false\n      # optional: Username to impersonate for the kubectl command\n      asUser: user\n      # optional: serviceAccount to use for the pod\n      serviceAccount: my-service-account\n      # optional: keep pod after exit (can be set to true for debugging)\n      keepPod: true\n      # optional: labels to add to the pod\n      labels:\n        key: value\n      # optional: annotations to add to the pod\n      annotations:\n        key: value\n      # optional: nodeSelector to add to the pod\n      nodeSelector:\n        key: value\n      # optional: resource limits to add to the pod\n      resources:\n        requests:\n          memory: \"64Mi\"\n          cpu: \"250m\"\n        limits:\n          memory: \"128Mi\"\n          cpu: \"500m\"\n\n      # optional: affinity to add to the pod\n      affinity:\n        # note: other types of affinity also supported\n        nodeAffinity:\n          requiredDuringSchedulingIgnoredDuringExecution:\n            nodeSelectorTerms:\n              - matchExpressions:\n                  - key: \"\u003ckey\u003e\"\n                    operator: \"\u003coperator\u003e\"\n                    values: [ \"\u003cvalue\u003e\" ]\n\n      # optional: tolerations to add to the pod\n      tolerations:\n        - key: \"\u003ckey\u003e\"\n          operator: \"\u003coperator\u003e\"\n          value: \"\u003cvalue\u003e\"\n          effect: \"\u003ceffect\u003e\"\n\n    # optional: clientID config (defaults to kafkactl-{username})\n    clientID: my-client-id\n\n    # optional: kafkaVersion (defaults to 2.5.0)\n    kafkaVersion: 1.1.1\n\n    # optional: timeout for admin requests (defaults to 3s)\n    requestTimeout: 10s\n\n    # optional: avro configuration\n    avro:\n      # optional: configure codec for (de)serialization as standard,avro (defaults to standard)\n      # see: https://github.com/deviceinsight/kafkactl/issues/123\n      jsonCodec: avro\n\n    # optional: schema registry\n    schemaRegistry:\n      url: localhost:8081\n\n      # optional: timeout for requests (defaults to 5s)\n      requestTimeout: 10s\n\n      # optional: basic auth credentials\n      username: admin\n      # optional: if omitted, password is looked up in the OS keyring or prompted interactively\n      password: admin\n\n      # optional: tls config for avro\n      tls:\n        enabled: true\n        ca: my-ca\n        cert: my-cert\n        certKey: my-key\n        # optional: passphrase for encrypted cert key (if omitted and key is encrypted, will prompt or use keyring)\n        certKeyPassphrase: my-passphrase\n        # set insecure to true to ignore all tls verification (defaults to false)\n        insecure: false\n\n    # optional: default protobuf messages search paths\n    protobuf:\n      importPaths:\n        - \"/usr/include/protobuf\"\n      protoFiles:\n        - \"someMessage.proto\"\n        - \"otherMessage.proto\"\n      protosetFiles:\n        - \"/usr/include/protoset/other.protoset\"\n      # optional: Protobuf JSON output options used by consume (all default to false)\n      # explicit --proto-marshal-option flags override these values\n      # see: https://pkg.go.dev/google.golang.org/protobuf@v1.36.6/encoding/protojson#MarshalOptions\n      marshalOptions:\n        allowPartial: true\n        useProtoNames: true\n        useEnumNumbers: true\n        emitUnpopulated: true\n        emitDefaultValues: true\n\n    producer:\n      # optional: changes the default partitioner\n      partitioner: \"hash\"\n\n      # optional: changes default required acks in produce request\n      # see: https://pkg.go.dev/github.com/IBM/sarama?utm_source=godoc#RequiredAcks\n      requiredAcks: \"WaitForAll\"\n\n      # optional: maximum permitted size of a message (defaults to 1000000)\n      maxMessageBytes: 1000000\n\n    consumer:\n      # optional: isolationLevel (defaults to ReadCommitted)\n      isolationLevel: ReadUncommitted\n\n# optional: keyring integration for credential storage (defaults to enabled)\nkeyring:\n  # set to false to disable OS keyring lookup and storage for passwords/passphrases\n  enabled: true\n----\n\n[#_config_file_read_order]\nThe config file location is resolved by\n\n. checking for a provided commandline argument: `--config-file=$PATH_TO_CONFIG`\n. evaluating the environment variable: `export KAFKA_CTL_CONFIG=$PATH_TO_CONFIG`\n. checking for a project config file in the working directory (see \u003c\u003c_project_config_files\u003e\u003e)\n. as default the config file is looked up from one of the following locations:\n ** `$HOME/.config/kafkactl/config.yml`\n ** `$HOME/.kafkactl/config.yml`\n ** `$APPDATA/kafkactl/config.yml`\n ** `/etc/kafkactl/config.yml`\n\n[#_project_config_files]\n==== Project config files\n\nIn addition to the config file locations above, _kafkactl_ allows to create a config file on project level.\nA project config file is meant to be placed at the root level of a git repo and declares the kafka configuration\nfor this repository/project.\n\nIn order to identify the config file as belonging to _kafkactl_ the following names can be used:\n\n* `kafkactl.yml`\n* `.kafkactl.yml`\n\nDuring initialization _kafkactl_ starts from the current working directory and recursively looks for a project level\nconfig file. The recursive lookup ends at the boundary of a git repository (i.e. if a `.git` folder is found).\nThis way, _kafkactl_ can be used conveniently anywhere in the git repository.\n\n[#_current_context]\n==== Current context\n\nThe current context can be set via commandline argument `--context`, environment variable `CURRENT_CONTEXT` or\nit can be defined in a file.\n\nIf no current context is defined, the first context in the config file is used as current context. Additionally,\nin this case a file storing the current context is created.\n\nThe file is typically stored next to the config file and named `current-context.yml`.\nThe location of the file can be overridden via environment variable `KAFKA_CTL_WRITABLE_CONFIG`.\n\n=== Auto completion\n\n==== bash\n\n----\nsource \u003c(kafkactl completion bash)\n----\n\nTo load completions for each session, execute once:\nLinux:\n\n----\nkafkactl completion bash \u003e /etc/bash_completion.d/kafkactl\n----\n\nMacOS:\n\n----\nkafkactl completion bash \u003e /usr/local/etc/bash_completion.d/kafkactl\n----\n\n==== zsh\n\nIf shell completion is not already enabled in your environment,\nyou will need to enable it. You can execute the following once:\n\n----\necho \"autoload -U compinit; compinit\" \u003e\u003e ~/.zshrc\n----\n\nTo load completions for each session, execute once:\n\n----\nkafkactl completion zsh \u003e \"${fpath[1]}/_kafkactl\"\n----\n\nYou will need to start a new shell for this setup to take effect.\n\n==== Fish\n\n----\nkafkactl completion fish | source\n----\n\nTo load completions for each session, execute once:\n\n----\nkafkactl completion fish \u003e ~/.config/fish/completions/kafkactl.fish\n----\n\n== Documentation\n\nThe documentation for all available commands can be found here:\n\nimage::https://img.shields.io/badge/command-docs-blue.svg[command docs,link=https://deviceinsight.github.io/kafkactl/]\n\n== Running in docker\n\nAssuming your Kafka brokers are accessible under `kafka1:9092` and `kafka2:9092`, you can list topics by running:\n\n[,bash]\n----\ndocker run --env BROKERS=\"kafka1:9092 kafka2:9092\" deviceinsight/kafkactl:latest get topics\n----\n\nIf a more elaborate config is needed, you can mount it as a volume:\n\n[,bash]\n----\ndocker run -v /absolute/path/to/config.yml:/etc/kafkactl/config.yml deviceinsight/kafkactl get topics\n----\n\n== Running in Kubernetes\n\n____\n:construction: This feature is still experimental.\n____\n\nIf your kafka cluster is not directly accessible from your machine, but it is accessible from a kubernetes cluster\nwhich in turn is accessible via `kubectl` from your machine you can configure kubernetes support:\n\n[,$yaml]\n----\ncontexts:\n  kafka-cluster:\n    brokers:\n      - broker1:9092\n      - broker2:9092\n    kubernetes:\n      enabled: true\n      binary: kubectl #optional\n      kubeContext: k8s-cluster\n      namespace: k8s-namespace\n----\n\nIf you are using `tlsSecret`, make sure that you set `ca`, `cert` and `certKey`. For example secret created by\nStrimzi Kafka operator will have values `ca.crt`, `user.crt`, `user.key`:\n\n[,$yaml]\n----\ncontexts:\n  kafka-cluster:\n    brokers:\n      - broker1:9092\n      - broker2:9092\n    kubernetes:\n      enabled: true\n      binary: kubectl #optional\n      kubeContext: k8s-cluster\n      namespace: k8s-namespace\n      tlsSecret: tls-secret\n    tls:\n      enabled: true\n      ca: ca.crt\n      cert: user.crt\n      certKey: user.key\n      # set insecure to true to ignore all tls verification (defaults to false)\n      insecure: false\n----\n\nIf your Kafka cluster requires SASL authentication, you can avoid passing credentials as plaintext environment\nvariables in the pod spec by using `saslSecret`. There are two options:\n\n. Reference an existing secret (must contain keys `username` and `password`):\n+\n[,$yaml]\n----\ncontexts:\n  kafka-cluster:\n    brokers:\n      - broker1:9092\n      - broker2:9092\n    kubernetes:\n      enabled: true\n      kubeContext: k8s-cluster\n      namespace: k8s-namespace\n      saslSecret:\n        name: my-sasl-secret\n    sasl:\n      enabled: true\n      mechanism: scram-sha256\n----\n\n. Let kafkactl create a short-lived secret based on credentials from the local config:\n+\n[,$yaml]\n----\ncontexts:\n  kafka-cluster:\n    brokers:\n      - broker1:9092\n      - broker2:9092\n    kubernetes:\n      enabled: true\n      kubeContext: k8s-cluster\n      namespace: k8s-namespace\n      saslSecret:\n        create: true\n    sasl:\n      enabled: true\n      mechanism: scram-sha256\n      username: my-user\n      password: my-password\n----\n+\nkafkactl will create the secret before the pod starts, verify no secret with the generated name already exists,\nand delete it once the pod exits.\n\nInstead of directly talking to kafka brokers a kafkactl docker image is deployed as a pod into the kubernetes\ncluster, and the defined namespace. Standard-Input and Standard-Output are then wired between the pod and your shell\nrunning kafkactl.\n\nThere are two options:\n\n. You can run `kafkactl attach` with your kubernetes cluster configured. This will use `kubectl run` to create a pod\nin the configured kubeContext/namespace which runs an image of kafkactl and gives you a `bash` into the container.\nStandard-in is piped to the pod and standard-out, standard-err directly to your shell. You even get auto-completion.\n. You can run any other kafkactl command with your kubernetes cluster configured. Instead of directly\nquerying the cluster a pod is deployed, and input/output are wired between pod and your shell.\n\nThe names of the brokers have to match the service names used to access kafka in your cluster. A command like this should\ngive you this information:\n\n[,bash]\n----\nkubectl get svc | grep kafka\n----\n\n____\n:bulb: The first option takes a bit longer to start up since an Ubuntu based docker image is used in order to have\na bash available. The second option uses a docker image build from scratch and should therefore be quicker.\nWhich option is more suitable, will depend on your use-case.\n____\n\n== Configuration via environment variables\n\nEvery key in the `config.yml` can be overwritten via environment variables. The corresponding environment variable\nfor a key can be found by applying the following rules:\n\n. replace `.` by `_`\n. replace `-` by `_`\n. write the key name in ALL CAPS\n\ne.g. the key `contexts.default.tls.certKey` has the corresponding environment variable `CONTEXTS_DEFAULT_TLS_CERTKEY`.\n\n*NOTE:* an array variable can be written using whitespace as delimiter. For example `BROKERS` can be provided as\n`BROKERS=\"broker1:9092 broker2:9092 broker3:9092\"`.\n\nIf environment variables for the `default` context should be set, the prefix `CONTEXTS_DEFAULT_` can be omitted.\nSo, instead of `CONTEXTS_DEFAULT_TLS_CERTKEY` one can also set `TLS_CERTKEY`.\nSee *root_test.go* for more examples.\n\nFor example, `PROTOBUF_MARSHALOPTIONS_EMITDEFAULTVALUES=true` overrides\n`contexts.default.protobuf.marshalOptions.emitDefaultValues`.\nThe other marshal options follow the same naming rules.\nThese options are also passed to the pod in Kubernetes mode.\nAn explicit consume flag such as `--proto-marshal-option emitDefaultValues=false`\noverrides both the configuration file and the environment variable.\n\n== Password Handling\n\nkafkactl supports multiple strategies for supplying passwords — for SASL authentication, TLS cert key passphrases, and Schema Registry credentials.\n\n=== Interactive Prompting\n\nIf a required credential is not present in the config file or environment, kafkactl prompts for it on the terminal:\n\n[,console]\n----\n$ kafkactl get topics\nSASL Password: _\n----\n\n=== OS Keyring\n\nAfter a password is entered interactively, kafkactl stores it in the OS keyring (GNOME Keyring on Linux, Keychain on macOS, Credential Manager on Windows) on successful command completion. Subsequent invocations retrieve the password from the keyring without prompting.\n\nKeyring storage is enabled by default. To disable it:\n\n[,yaml]\n----\nkeyring:\n  enabled: false\n----\n\nTo clear a stored credential and re-prompt (e.g. after a password rotation), use the `--clear-keyring` flag:\n\n[,console]\n----\n$ kafkactl get topics --clear-keyring\n----\n\nThis deletes the existing keyring entry before prompting and stores the newly entered value on success.\n\n=== Credentials in Config File\n\nCredentials can be stored directly in the config file. This is convenient for development but not recommended for production environments:\n\n[,yaml]\n----\ncontexts:\n  my-cluster:\n    sasl:\n      enabled: true\n      username: my-user\n      password: my-password   # stored in plaintext\n    tls:\n      certKey: my-key.pem\n      certKeyPassphrase: my-passphrase   # stored in plaintext\n----\n\n=== Kubernetes Secrets\n\nWhen running in Kubernetes mode, use `saslSecret` to avoid passing credentials as plaintext environment variables in the pod.\nSee \u003c\u003cRunning in Kubernetes\u003e\u003e for the full `saslSecret` configuration options.\n\n== Plugins\n\n_kafkactl_ supports plugins to cope with specifics when using Kafka-compatible clusters available from cloud providers such as Azure or AWS.\n\nAt the moment, plugins can only be used to implement a `tokenProvider` for _oauth_ authentication.\nIn the future, plugins might implement additional commands to query data or configuration which is not part of the Kafka-API. One example would be Eventhub consumer groups/offsets for Azure.\n\nSee the plugin documentation for additional documentation and usage examples.\n\nAvailable plugins:\n\n* https://github.com/deviceinsight/kafkactl-plugins/blob/main/aws/README.adoc[aws plugin]\n* https://github.com/deviceinsight/kafkactl-plugins/blob/main/azure/README.adoc[azure plugin]\n\n=== Generic Token Provider\n\nFor cases where a custom plugin is not needed, _kafkactl_ provides a built-in `generic` token provider that executes a script to retrieve OAuth tokens dynamically.\n\n==== Configuration\n\n[,yaml]\n----\ncontexts:\n  my-cluster:\n    sasl:\n      enabled: true\n      mechanism: oauth\n      tokenprovider:\n        plugin: generic\n        options:\n          script: /path/to/get-token.sh\n          args:\n            - intended-scope\n----\n\n==== Script Output Format\n\nThe script must output a JSON object with the following structure:\n\n[,json]\n----\n{\n  \"token\": \"your-oauth-access-token\",\n  \"extensions\": {\n    \"key\": \"value\"\n  }\n}\n----\n\n* `token` (required): The OAuth access token string\n* `extensions` (optional): A map of string key-value pairs for OAuth extensions\n\n==== Example Script\n\n[,bash]\n----\n#!/bin/bash\n# Example script that fetches a token from an OAuth provider\nSCOPE=${1:-kafka}\n\ncurl -s --fail -X POST \"$AUTH_SERVER/oauth/token\" \\\n  -H \"Content-Type: application/x-www-form-urlencoded\" \\\n  -d \"grant_type=client_credentials\" \\\n  -d \"client_id=$CLIENT_ID\" \\\n  -d \"client_secret=$CLIENT_SECRET\" \\\n  -d \"scope=$SCOPE\" | jq -r '{\"token\": .access_token}'\n----\n\nThe script is executed each time a token is needed, allowing for automatic token refresh.\n\n== Examples\n\n=== Consuming messages\n\nConsuming messages from a topic can be done with:\n\n[,bash]\n----\nkafkactl consume my-topic\n----\n\nIn order to consume starting from the oldest offset use:\n\n[,bash]\n----\nkafkactl consume my-topic --from-beginning\n----\n\nThe following example prints message `key` and `timestamp` as well as `partition` and `offset` in `yaml` format:\n\n[,bash]\n----\nkafkactl consume my-topic --print-keys --print-timestamps -o yaml\n----\n\nTo print partition in default output format use:\n\n[,bash]\n----\nkafkactl consume my-topic --print-partitions\n----\n\nHeaders of kafka messages can be printed with the parameter `--print-headers` e.g.:\n\n[,bash]\n----\nkafkactl consume my-topic --print-headers -o yaml\n----\n\nIf one is only interested in the last `n` messages this can be achieved by `--tail` e.g.:\n\n[,bash]\n----\nkafkactl consume my-topic --tail=5\n----\n\nThe consumer can be stopped when the latest offset is reached using `--exit` parameter e.g.:\n\n[,bash]\n----\nkafkactl consume my-topic --from-beginning --exit\n----\n\nThe consumer can compute the offset it starts from using a timestamp:\n\n[,bash]\n----\nkafkactl consume my-topic --from-timestamp 1384216367189\nkafkactl consume my-topic --from-timestamp 2014-04-26T17:24:37.123Z\nkafkactl consume my-topic --from-timestamp 2014-04-26T17:24:37.123\nkafkactl consume my-topic --from-timestamp 2009-08-12T22:15:09Z\nkafkactl consume my-topic --from-timestamp 2017-07-19T03:21:51\nkafkactl consume my-topic --from-timestamp 2013-04-01T22:43\nkafkactl consume my-topic --from-timestamp 2014-04-26\n----\n\nThe `from-timestamp` parameter supports different timestamp formats. It can either be a number representing the epoch milliseconds\nor a string with a timestamp in one of the https://github.com/deviceinsight/kafkactl/blob/main/internal/util/util.go#L10[supported date formats].\n\n*NOTE:* `--from-timestamp` is not designed to schedule the beginning of consumer's consumption. The offset corresponding to the timestamp is computed at the beginning of the process. So if you set it to a date in the future, the consumer will start from the latest offset.\n\nThe consumer can be stopped when the offset corresponding to a particular timestamp is reached:\n\n[,bash]\n----\nkafkactl consume my-topic --from-timestamp 2017-07-19T03:30:00 --to-timestamp 2017-07-19T04:30:00\n----\n\nThe `to-timestamp` parameter supports the same formats as `from-timestamp`.\n\n*NOTE:* `--to-timestamp` is not designed to schedule the end of consumer's consumption. The offset corresponding to the timestamp is computed at the beginning of the process. So if you set it to a date in the future, the consumer will stop at the current latest offset.\n\nThe following example prints keys in hex and values in base64:\n\n[,bash]\n----\nkafkactl consume my-topic --print-keys --key-encoding=hex --value-encoding=base64\n----\n\nThe consumer can convert protobuf messages to JSON in keys (optional) and values:\n\n[,bash]\n----\nkafkactl consume my-topic --value-proto-type MyTopicValue --key-proto-type MyTopicKey --proto-file kafkamsg.proto\n----\n\nBy default the consumer aborts as soon as a message cannot be deserialized (for example an invalid protobuf or avro\npayload on the topic). To keep consuming and skip such messages, printing a warning for each one instead, use\n`--ignore-errors`:\n\n[,bash]\n----\nkafkactl consume my-topic --value-proto-type MyTopicValue --proto-file kafkamsg.proto --ignore-errors\n----\n\nTo join a consumer group and consume messages as a member of the group:\n\n[,bash]\n----\nkafkactl consume my-topic --group my-consumer-group\n----\n\nIf you want to limit the number of messages that will be read, specify `--max-messages`:\n\n[,bash]\n----\nkafkactl consume my-topic --max-messages 2\n----\n\nMessages can be filtered by key, value, or headers using glob patterns:\n\n[,bash]\n----\n# filter by key pattern\nkafkactl consume my-topic --filter-key \"user-*\" --print-keys\n\n# filter by value pattern\nkafkactl consume my-topic --filter-value \"error:*\"\n\n# filter by header pattern\nkafkactl consume my-topic --filter-header \"trace-id=abc-*\" --print-headers\n\n# combine multiple filters (all must match)\nkafkactl consume my-topic --filter-key \"user-*\" --filter-value \"*error*\" --filter-header \"env=prod\"\n----\n\nGlob patterns support the following wildcards:\n\n* `*` - matches any sequence of characters\n* `?` - matches a single character\n* `[abc]` - matches any character in the set\n* `[!abc]` - matches any character not in the set\n* `{a,b,c}` - matches any of the alternatives\n\nExamples of glob patterns:\n\n[,bash]\n----\n# match keys starting with \"user-\"\n--filter-key \"user-*\"\n\n# match keys for user or admin\n--filter-key \"{user,admin}-*\"\n\n# match values containing \"error\" or \"warn\"\n--filter-value \"*error*\" or --filter-value \"*warn*\"\n\n# match specific header patterns\n--filter-header \"trace-id=abc-???\"\n----\n\n=== Producing messages\n\nProducing messages can be done in multiple ways. If we want to produce a message with `key='my-key'`,\n`value='my-value'` to the topic `my-topic` this can be achieved with one of the following commands:\n\n[,bash]\n----\necho \"my-key#my-value\" | kafkactl produce my-topic --separator=#\necho \"my-value\" | kafkactl produce my-topic --key=my-key\nkafkactl produce my-topic --key=my-key --value=my-value\n----\n\nIf we have a file containing messages where each line contains `key` and `value` separated by `#`, the file can be\nused as input to produce messages to topic `my-topic`:\n\n[,bash]\n----\ncat myfile | kafkactl produce my-topic --separator=#\n----\n\nThe same can be accomplished without piping the file to stdin with the `--file` parameter:\n\n[,bash]\n----\nkafkactl produce my-topic --separator=# --file=myfile\n----\n\nIf the messages in the input file need to be split by a different delimiter than `\\n` a custom line separator can be provided:\n\n[,bash]\n----\nkafkactl produce my-topic --separator=# --lineSeparator=|| --file=myfile\n----\n\n*NOTE:* if the file was generated with `kafkactl consume --print-keys --print-timestamps my-topic` the produce\ncommand is able to detect the message timestamp in the input and will ignore it.\n\nIt is also possible to produce messages in json format:\n\n[,bash]\n----\n# each line in myfile.json is expected to contain a json object with fields key, value and headers\nkafkactl produce my-topic --file=myfile.json --input-format=json\ncat myfile.json | kafkactl produce my-topic --input-format=json\necho '{\"value\": \"my-value\"}' | kafkactl produce my-topic --input-format=json\necho '{\"key\": \"my-key\", \"value\": \"my-value\", \"headers\": {\"my-header\": \"val\"}}' | kafkactl produce my-topic --input-format=json\n----\n\nthe number of messages produced per second can be controlled with the `--rate` parameter:\n\n[,bash]\n----\ncat myfile | kafkactl produce my-topic --separator=# --rate=200\n----\n\nIt is also possible to specify the partition to insert the message:\n\n[,bash]\n----\nkafkactl produce my-topic --key=my-key --value=my-value --partition=2\n----\n\nAdditionally, a different partitioning scheme can be used. When a `key` is provided the default partitioner\nuses the `hash` of the `key` to assign a partition. So the same `key` will end up in the same partition:\n\n[,bash]\n----\n# the following 3 messages will all be inserted to the same partition\nkafkactl produce my-topic --key=my-key --value=my-value\nkafkactl produce my-topic --key=my-key --value=my-value\nkafkactl produce my-topic --key=my-key --value=my-value\n\n# the following 3 messages will probably be inserted to different partitions\nkafkactl produce my-topic --key=my-key --value=my-value --partitioner=random\nkafkactl produce my-topic --key=my-key --value=my-value --partitioner=random\nkafkactl produce my-topic --key=my-key --value=my-value --partitioner=random\n----\n\nMessage headers can also be written:\n\n[,bash]\n----\nkafkactl produce my-topic --key=my-key --value=my-value --header key1:value1 --header key2:value\\:2\n----\n\nThe following example writes the key from base64 and value from hex:\n\n[,bash]\n----\nkafkactl produce my-topic --key=dGVzdC1rZXk= --key-encoding=base64 --value=0000000000000000 --value-encoding=hex\n----\n\nYou can control how many replica acknowledgements are needed for a response:\n\n[,bash]\n----\nkafkactl produce my-topic --key=my-key --value=my-value --required-acks=WaitForAll\n----\n\nProducing null values (tombstone record) is also possible:\n\n[,bash]\n----\n kafkactl produce my-topic --null-value\n----\n\nProducing protobuf message converted from JSON:\n\n[,bash]\n----\nkafkactl produce my-topic --key='{\"keyField\":123}' --key-proto-type MyKeyMessage --value='{\"valueField\":\"value\"}' --value-proto-type MyValueMessage --proto-file kafkamsg.proto\n----\n\nA more complex protobuf message converted from a multi-line JSON string can be produced using a file input with custom separators.\n\nFor example, if you have the following protobuf definition (`complex.proto`):\n\n[,protobuf]\n----\nsyntax = \"proto3\";\n\nimport \"google/protobuf/timestamp.proto\";\n\nmessage ComplexMessage {\n  CustomerInfo customer_info = 1;\n  DeviceInfo device_info = 2;\n}\n\nmessage CustomerInfo {\n  string customer_id = 1;\n  string name = 2;\n}\n\nmessage DeviceInfo {\n  string serial = 1;\n  google.protobuf.Timestamp last_update  = 2;\n}\n----\n\nAnd you have the following file (`complex-msg.txt`) that contains the key and value of the message:\n\n[,text]\n----\nmsg-key##\n{\n    \"customer_info\": {\n        \"customer_id\": \"12345\",\n        \"name\": \"Bob\"\n    },\n    \"device_info\": {\n        \"serial\": \"abcde\",\n        \"last_update\": \"2024-03-02T07:01:02.000Z\"\n    }\n}\n+++\n----\n\nThe command to produce the protobuf message using sample protobuf definition and input file would be:\n\n[,bash]\n----\nkafkactl produce my-topic --value-proto-type=ComplexMessage --proto-file=complex.proto --lineSeparator='+++' --separator='##' --file=complex-msg.txt\n----\n\n=== Avro support\n\nIn order to enable avro support you just have to add the schema registry to your configuration:\n\n[,$yaml]\n----\ncontexts:\n  localhost:\n    schemaRegistry:\n      url: localhost:8081\n----\n\n==== Producing to an avro topic\n\n`kafkactl` will lookup the topic in the schema registry in order to determine if key or value needs to be avro encoded.\nIf producing with the latest `schemaVersion` is sufficient, no additional configuration is needed an `kafkactl` handles\nthis automatically.\n\nIf however one needs to produce an older `schemaVersion` this can be achieved by providing the parameters `keySchemaVersion`, `valueSchemaVersion`.\n\n===== Example\n\n[,bash]\n----\n# create a topic\nkafkactl create topic avro_topic\n# add a schema for the topic value\ncurl -X POST -H \"Content-Type: application/vnd.schemaregistry.v1+json\" \\\n--data '{\"schema\": \"{\\\"type\\\": \\\"record\\\", \\\"name\\\": \\\"LongList\\\", \\\"fields\\\" : [{\\\"name\\\": \\\"next\\\", \\\"type\\\": [\\\"null\\\", \\\"LongList\\\"], \\\"default\\\": null}]}\"}' \\\nhttp://localhost:8081/subjects/avro_topic-value/versions\n# produce a message\nkafkactl produce avro_topic --value {\\\"next\\\":{\\\"next\\\":{}}}\n# consume the message\nkafkactl consume avro_topic --from-beginning --print-schema -o yaml\n----\n\n==== Consuming from an avro topic\n\nAs for producing `kafkactl` will also lookup the topic in the schema registry to determine if key or value needs to be\ndecoded with an avro schema.\n\nThe `consume` command handles this automatically and no configuration is needed.\n\nAn additional parameter `print-schema` can be provided to display the schema used for decoding.\n\n=== JSON schema support\n\nJSON schema support works similarly to Avro. When a schema registry is configured, `kafkactl` will automatically detect\ntopics with JSON schemas registered and encode/decode messages using the Confluent wire format.\n\n==== Producing to a JSON schema topic\n\n`kafkactl` will lookup the topic in the schema registry in order to determine if key or value has a JSON schema registered.\nThe input JSON is validated against the schema before producing. If producing with the latest `schemaVersion` is sufficient,\nno additional configuration is needed and `kafkactl` handles this automatically.\n\nIf however one needs to produce an older `schemaVersion` this can be achieved by providing the parameters `keySchemaVersion`, `valueSchemaVersion`.\n\n===== Example\n\n[,bash]\n----\n# create a topic\nkafkactl create topic json_topic\n# add a JSON schema for the topic value\ncurl -X POST -H \"Content-Type: application/vnd.schemaregistry.v1+json\" \\\n--data '{\"schemaType\": \"JSON\", \"schema\": \"{\\\"type\\\": \\\"object\\\", \\\"properties\\\": {\\\"name\\\": {\\\"type\\\": \\\"string\\\"}, \\\"age\\\": {\\\"type\\\": \\\"integer\\\"}}, \\\"required\\\": [\\\"name\\\", \\\"age\\\"]}\"}' \\\nhttp://localhost:8081/subjects/json_topic-value/versions\n# produce a message\nkafkactl produce json_topic --value '{\"name\":\"Alice\",\"age\":30}'\n# consume the message\nkafkactl consume json_topic --from-beginning --print-schema -o yaml\n----\n\n==== Consuming from a JSON schema topic\n\nAs for producing, `kafkactl` will also lookup the topic in the schema registry to determine if key or value needs to be\ndecoded with a JSON schema.\n\nThe `consume` command handles this automatically and no configuration is needed.\n\nAn additional parameter `print-schema` can be provided to display the schema used for decoding.\n\n=== Protobuf support\n\n`kafkactl` can consume and produce protobuf-encoded messages. In order to enable protobuf serialization/deserialization\nyou should add flag `--value-proto-type` and optionally `--key-proto-type` (if keys encoded in protobuf format)\nwith type name. Protobuf-encoded messages are mapped with https://developers.google.com/protocol-buffers/docs/proto3#json[pbjson].\n\n`kafkactl` will search messages in following order:\n\n. Protoset files specified in `--protoset-file` flag\n. Protoset files specified in `context.protobuf.protosetFiles` config value\n. Proto files specified in `--proto-file` flag\n. Proto files specified in `context.protobuf.protoFiles` config value\n\nProto files may require some dependencies in `import` sections. To specify additional lookup paths use\n`--proto-import-path` flag or `context.protobuf.importPaths` config value.\n\nIf provided message types was not found `kafkactl` will return error.\n\nNote that if you want to use raw proto files `protoc` installation don't need to be installed.\n\nAlso note that protoset files must be compiled with included imports:\n\n[,bash]\n----\nprotoc -o kafkamsg.protoset --include_imports kafkamsg.proto\n----\n\n==== Example\n\nAssume you have following proto schema in `kafkamsg.proto`:\n\n[,protobuf]\n----\nsyntax = \"proto3\";\n\nimport \"google/protobuf/timestamp.proto\";\n\nmessage TopicMessage {\n  google.protobuf.Timestamp produced_at = 1;\n  int64 num = 2;\n}\n\nmessage TopicKey {\n  float fvalue = 1;\n}\n----\n\n\"well-known\" `google/protobuf` types are included so no additional proto files needed.\n\nTo produce message run\n\n[,bash]\n----\nkafkactl produce \u003ctopic\u003e --key '{\"fvalue\":1.2}' --key-proto-type TopicKey --value '{\"producedAt\":\"2021-12-01T14:10:12Z\",\"num\":\"1\"}' --value-proto-type TopicValue --proto-file kafkamsg.proto\n----\n\nor with protoset\n\n[,bash]\n----\nkafkactl produce \u003ctopic\u003e --key '{\"fvalue\":1.2}' --key-proto-type TopicKey --value '{\"producedAt\":\"2021-12-01T14:10:12Z\",\"num\":\"1\"}' --value-proto-type TopicValue --protoset-file kafkamsg.protoset\n----\n\nTo consume messages run\n\n[,bash]\n----\nkafkactl consume \u003ctopic\u003e --key-proto-type TopicKey --value-proto-type TopicValue --proto-file kafkamsg.proto\n----\n\nor with protoset\n\n[,bash]\n----\nkafkactl consume \u003ctopic\u003e --key-proto-type TopicKey --value-proto-type TopicValue --protoset-file kafkamsg.protoset\n----\n\n\n\n=== Topic management\n\n==== List topics\n\nIn order to get a list of topics the `get topics` command can be used:\n\n[,bash]\n----\nkafkactl get topics\nkafkactl list topics\n----\n\n==== Describe topic\n\nA detailed description of a topic can be obtained with `describe topic`:\n\n[,bash]\n----\nkafkactl describe topic my-topic\n----\n\nPer default only overwritten config entries are printed. To print all config entries including defaults use:\n\n[,bash]\n----\nkafkactl describe topic my-topic --all-configs\n----\n\nTo print only partition details of partitions with messages use:\n[,bash]\n----\nkafkactl describe topic my-topic --skip-empty\n----\n\n==== Create topic\n\nThe `create topic` allows you to create one or multiple topics.\n\nBasic usage:\n[,bash]\n----\nkafkactl create topic my-topic\n----\n\nThe partition count can be specified with:\n[,bash]\n----\nkafkactl create topic my-topic --partitions 32\n----\n\nThe replication factor can be specified with:\n[,bash]\n----\nkafkactl create topic my-topic --replication-factor 3\n----\n\nConfigs can also be provided:\n[,bash]\n----\nkafkactl create topic my-topic --config retention.ms=3600000 --config=cleanup.policy=compact\n----\n\nThe topic configuration can also be taken from an existing topic using the following:\n[,bash]\n----\nkafkactl describe topic my-topic -o json \u003e my-topic-config.json\nkafkactl create topic my-topic-clone --file my-topic-config.json\n----\n\n==== Altering topics\n\nUsing the `alter topic` command allows you to change the partition count, replication factor and topic-level\nconfigurations of an existing topic.\n\nThe partition count can be increased with:\n\n[,bash]\n----\nkafkactl alter topic my-topic --partitions 32\n----\n\nThe replication factor can be altered with:\n\n[,bash]\n----\nkafkactl alter topic my-topic --replication-factor 2\n----\n\n____\n:information_source: when altering replication factor, kafkactl tries to keep the number of replicas assigned to each\nbroker balanced. If you need more control over the assigned replicas use `alter partition` directly.\n____\n\nThe topic configs can be edited by supplying key value pairs as follows:\n\n[,bash]\n----\nkafkactl alter topic my-topic --config retention.ms=3600000 --config cleanup.policy=compact\n----\n\n____\n:bulb: use the flag `--validate-only` to perform a dry-run without actually modifying the topic\n____\n\n==== Altering partitions\n\nThe assigned replicas of a partition can directly be altered with:\n\n[,bash]\n----\n# set brokers 102,103 as replicas for partition 3 of topic my-topic\nkafkactl alter partition my-topic 3 -r 102,103\n----\n\n==== Clone topic\n\nNew topic may be created from existing topic as follows:\n\n[,bash]\n----\nkafkactl clone topic source-topic target-topic\n----\n\nSource topic must exist, target topic must not exist.\n`kafkactl` clones partitions count, replication factor and config entries.\n\n\n==== Delete Records from a topics\n\nCommand to be used to delete records from partition, which have an offset smaller than the provided offset.\n\n[,bash]\n----\n# delete records with offset \u003c 123 from partition 0 and offset \u003c 456 from partition 1\nkafkactl delete records my-topic --offset 0=123 --offset 1=456\n----\n\n\n=== Consumer group management\n\n==== List Consumer groups\n\nIn order to get a list of consumer groups the `get consumer-groups` command can be used:\n\n[,bash]\n----\n# all available consumer groups\nkafkactl get consumer-groups\n# only consumer groups for a single topic\nkafkactl get consumer-groups --topic my-topic\n# using command alias\nkafkactl get cg\n----\n\nTo get detailed information about the consumer group use `describe consumer-group`. If the parameter `--partitions`\nis provided details will be printed for each partition otherwise the partitions are aggregated to the clients.\n\n==== Describe Consumer group\n\n[,bash]\n----\n# describe a consumer group\nkafkactl describe consumer-group my-group\n# show partition details only for partitions with lag\nkafkactl describe consumer-group my-group --only-with-lag\n# show details only for a single topic\nkafkactl describe consumer-group my-group --topic my-topic\n# using command alias\nkafkactl describe cg my-group\n----\n\n==== Create consumer groups\n\nA consumer-group can be created as follows:\n\n[,bash]\n----\n# create group with offset for all partitions set to oldest\nkafkactl create consumer-group my-group --topic my-topic --oldest\n# create group with offset for all partitions set to newest\nkafkactl create consumer-group my-group --topic my-topic --newest\n# create group with offset for a single partition set to specific offset\nkafkactl create consumer-group my-group --topic my-topic --partition 5 --offset 100\n# create group for multiple topics with offset for all partitions set to oldest\nkafkactl create consumer-group my-group --topic my-topic-a --topic my-topic-b --oldest\n----\n\n==== Clone consumer group\n\nA consumer group may be created as clone of another consumer group as follows:\n\n[,bash]\n----\nkafkactl clone consumer-group source-group target-group\n----\n\nSource group must exist and have committed offsets. Target group must not exist or don't have committed offsets.\n`kafkactl` clones topic assignment and partition offsets.\n\n==== Reset consumer group offsets\n\nin order to ensure the reset does what it is expected, per default only\nthe results are printed without actually executing it. Use the additional parameter `--execute` to perform the reset.\n\n[,bash]\n----\n# reset offset of for all partitions to oldest offset\nkafkactl reset offset my-group --topic my-topic --oldest\n# reset offset of for all partitions to newest offset\nkafkactl reset offset my-group --topic my-topic --newest\n# reset offset for a single partition to specific offset\nkafkactl reset offset my-group --topic my-topic --partition 5 --offset 100\n# reset offset to newest for all topics in the group\nkafkactl reset offset my-group --all-topics --newest\n# reset offset of for all partitions on multiple topics to oldest offset\nkafkactl reset offset my-group --topic my-topic-a --topic my-topic-b --oldest\n# reset offset to offset at a given timestamp(epoch)/datetime\nkafkactl reset offset my-group --topic my-topic-a --to-datetime 2014-04-26T17:24:37.123Z\n# reset offset to offset at a given timestamp(epoch)/datetime\nkafkactl reset offset my-group --topic my-topic-a --to-datetime 1697726906352\n----\n\n==== Delete consumer group offsets\n\nIn order to delete a consumer group offset use `delete offset`\n\n[,bash]\n----\n# delete offset for all partitions of topic my-topic\nkafkactl delete offset my-group --topic my-topic\n# delete offset for partition 1 of topic my-topic\nkafkactl delete offset my-group --topic my-topic --partition 1\n----\n\n==== Delete consumer groups\n\nIn order to delete a consumer group or a list of consumer groups use `delete consumer-group`\n\n[,bash]\n----\n# delete consumer group my-group\nkafkactl delete consumer-group my-group\n----\n\n=== ACL Management\n\nAvailable ACL operations are documented https://docs.confluent.io/platform/current/kafka/authorization.html#operations[here].\n\n==== Create a new ACL\n\n[,bash]\n----\n# create an acl that allows topic read for a user 'consumer'\nkafkactl create acl --topic my-topic --operation read --principal User:consumer --allow\n# create an acl that denies topic write for a user 'consumer' coming from a specific host\nkafkactl create acl --topic my-topic --operation write --host 1.2.3.4 --principal User:consumer --deny\n# allow multiple operations\nkafkactl create acl --topic my-topic --operation read --operation describe --principal User:consumer --allow\n# allow on all topics with prefix common prefix\nkafkactl create acl --topic my-prefix --pattern prefixed --operation read --principal User:consumer --allow\n----\n\n==== List ACLs\n\n[,bash]\n----\n# list all acl\nkafkactl get acl\n# list all acl (alias command)\nkafkactl get access-control-list\n# filter only topic resources\nkafkactl get acl --topics\n# filter only consumer group resources with operation read\nkafkactl get acl --groups --operation read\n# filter specific topic and user\nkafkactl get acl --resource-name my-topic --principal User:myUser\n# filter specific topic and host\nkafkactl get acl --resource-name my-topic --host my-host\n----\n\n==== Delete ACLs\n\n[,bash]\n----\n# delete all topic read acls\nkafkactl delete acl --topics --operation read --pattern any\n# delete all topic acls for any operation\nkafkactl delete acl --topics --operation any --pattern any\n# delete all cluster acls for any operation\nkafkactl delete acl --cluster --operation any --pattern any\n# delete all consumer-group acls with operation describe, patternType prefixed and permissionType allow\nkafkactl delete acl --groups --operation describe --pattern prefixed --allow\n# delete all topic acls for a principal\nkafkactl delete acl --topics --operation any --pattern any --prinicipal User:myUser\n# delete all topic acls for a resource name\nkafkactl delete acl --topics --operation any --pattern any --resource-name my-topic\n# delete all topic acls for a host\nkafkactl delete acl --topics --operation any --pattern any --host my-host\n----\n\n=== Broker Management\n\n==== Getting Brokers\n\nTo get the list of brokers of a kafka cluster use `get brokers`\n\n[,bash]\n----\n# get the list of brokers\nkafkactl get brokers\n----\n\n==== Describe Broker\n\nTo view configs for a single broker use `describe broker`\n\n[,bash]\n----\n# describe broker\nkafkactl describe broker 1\n----\n\nPer default only dynamic configs are shown. To view all configs use `--all-configs`\n\n[,bash]\n----\nkafkactl describe broker 1 --all-configs\n----\n\nAdditionally, only default configs can be shown with:\n[,bash]\n----\nkafkactl describe broker default\n----\n\n==== Altering brokers\n\nUsing the `alter broker` command allows you to change dynamic broker configurations for individual brokers or cluster-wide defaults.\n\nTo alter a configuration for a specific broker:\n\n[,bash]\n----\nkafkactl alter broker 101 --config background.threads=8\n----\n\nTo alter a cluster-wide default configuration (affects brokers without individual overrides):\n\n[,bash]\n----\nkafkactl alter broker default --config background.threads=8\n----\n\nMultiple configurations can be altered simultaneously:\n\n[,bash]\n----\nkafkactl alter broker 101 --config background.threads=8 --config log.cleaner.threads=2\n----\n\n____\n:bulb: use the flag `--validate-only` to perform a dry-run without actually modifying the broker configuration\n____\n\n____\n:information_source: only dynamically configurable broker properties can be altered. Static properties like `broker.id` or `log.dirs` require a broker restart to change.\n____\n\n\n=== SCRAM User Management\n\nkafkactl provides comprehensive SCRAM (Salted Challenge Response Authentication Mechanism) user management capabilities for Kafka clusters that support SCRAM authentication. This allows you to create, modify, and manage user credentials directly through kafkactl.\n\n==== Requirements\n\n* Kafka 2.7.0+ (for SCRAM user management APIs)\n* Admin privileges on the Kafka cluster\n* SCRAM-enabled listeners configured on Kafka brokers\n\n==== Supported Mechanisms\n\n* `SCRAM-SHA-256` (default)\n* `SCRAM-SHA-512`\n\n==== Create SCRAM Users\n\nCreate a new SCRAM user with default settings:\n\n[,bash]\n----\n# Create user with SCRAM-SHA-256 (default mechanism)\nkafkactl create user myuser --password mypassword\n\n# Create user with specific mechanism\nkafkactl create user myuser --password mypassword --mechanism SCRAM-SHA-512\n\n# Create user with custom iterations (default: 4096)\nkafkactl create user myuser --password mypassword --iterations 8192\n\n# Create user with custom base64-encoded salt\nkafkactl create user myuser --password mypassword --salt \"c2FsdA==\"\n----\n\n==== Alter SCRAM Users\n\nUpdate existing user credentials:\n\n[,bash]\n----\n# Update user password (keeps existing mechanism)\nkafkactl alter user myuser --password newpassword\n\n# Update user with different mechanism\nkafkactl alter user myuser --password newpassword --mechanism SCRAM-SHA-512\n\n# Update with custom iterations\nkafkactl alter user myuser --password newpassword --iterations 16384\n----\n\n==== Delete SCRAM Users\n\nRemove SCRAM credentials by mechanism:\n\n[,bash]\n----\n# Delete SCRAM-SHA-256 credentials (default)\nkafkactl delete user myuser --mechanism SCRAM-SHA-256\n\n# Delete SCRAM-SHA-512 credentials\nkafkactl delete user myuser --mechanism SCRAM-SHA-512\n\n# Note: A user may have multiple mechanisms, delete each separately\n----\n\n==== List SCRAM Users\n\nGet all SCRAM users and their mechanisms:\n\n[,bash]\n----\n# List all users (table format)\nkafkactl get users\n\n# List users in JSON format\nkafkactl get users -o json\n\n# List users in YAML format\nkafkactl get users -o yaml\n----\n\n==== Describe SCRAM User\n\nGet detailed information about a specific user:\n\n[,bash]\n----\n# Describe user (table format)\nkafkactl describe user myuser\n\n# Describe user in JSON format\nkafkactl describe user myuser -o json\n\n# Describe user in YAML format\nkafkactl describe user myuser -o yaml\n----\n\n==== Security Considerations\n\n* **Salt Generation**: kafkactl automatically generates cryptographically secure random salts unless custom salts are provided\n* **Password Security**: Passwords are transmitted securely to Kafka and never stored by kafkactl\n* **Mechanism Support**: Users can have credentials for multiple SCRAM mechanisms simultaneously\n* **Admin Privileges**: SCRAM user management requires admin-level access to the Kafka cluster\n\n==== Multi-Mechanism Users\n\nA single user can have credentials for multiple SCRAM mechanisms:\n\n[,bash]\n----\n# Create user with SCRAM-SHA-256\nkafkactl create user myuser --password mypassword --mechanism SCRAM-SHA-256\n\n# Add SCRAM-SHA-512 credentials to the same user\nkafkactl create user myuser --password mypassword --mechanism SCRAM-SHA-512\n\n# User now has both mechanisms\nkafkactl describe user myuser\n----\n\n== Development\n\nIn order to see linter errors before commit, add the following pre-commit hook:\n\n[,bash]\n----\npip install --user pre-commit\npre-commit install\n----\n","project_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdeviceinsight%2Fkafkactl","html_url":"https://awesome.ecosyste.ms/projects/github.com%2Fdeviceinsight%2Fkafkactl","lists_url":"https://awesome.ecosyste.ms/api/v1/projects/github.com%2Fdeviceinsight%2Fkafkactl/lists"}