diff --git a/.pipeline/kafka-setup/blubber.yaml b/.pipeline/kafka-setup/blubber.yaml index 2f191e9bc..9b65ba359 100644 --- a/.pipeline/kafka-setup/blubber.yaml +++ b/.pipeline/kafka-setup/blubber.yaml @@ -1,103 +1,108 @@ # syntax=docker-registry.wikimedia.org/repos/releng/blubber:v0.17.0 version: v4 base: docker-registry.wikimedia.org/openjdk-11-jdk lives: in: /datahub variants: datahub-source: base: docker-registry.wikimedia.org/bullseye:latest apt: packages: [ 'git', 'ca-certificates' ] builders: - custom: command: - git - clone - '--branch' - 'v0.10.0' - '--single-branch' - "https://github.com/datahub-project/datahub.git" - "/datahub" - custom: command: - chmod - +x - /datahub/docker/kafka-setup/kafka-setup.sh - /datahub/docker/kafka-setup/kafka-config.sh - /datahub/docker/kafka-setup/kafka-topic-workers.sh cub: base: docker-registry.wikimedia.org/bullseye:latest lives: in: "/confluent-data-utils" apt: packages: [ 'git', 'ca-certificates', python3-pip ] builders: - custom: command: - git - clone - '--branch' - 'v0.0.60' - '--single-branch' - "https://github.com/confluentinc/confluent-docker-utils.git" - "/confluent-data-utils" - custom: command: [ 'pip', 'install', '--prefer-binary', '--prefix', '/var/tmp', '--upgrade', '/confluent-data-utils' ] dockerize: base: docker-registry.wikimedia.org/golang1.19:1.19-1-20230625 builder: command: [ 'go', 'install', 'github.com/jwilder/dockerize@v0.6.1'] shell: builders: - custom: - command: [ 'cp', '/home/runuser/.bashrc', '/tmp/.bashrc'] + command: [ '/bin/sh', '-c', 'echo "#!/bin/bash\n/usr/bin/kafka-topics \"\044@\"" > /tmp/kafka-topics.sh'] - custom: - command: [ 'sed', '-i', '$ aalias kafka-topics.sh=/usr/bin/kafka-topics', '/tmp/.bashrc'] + command: [ '/bin/sh', '-c', 'echo "#!/bin/bash\n/usr/bin/kafka-configs \"\044@\"" > /tmp/kafka-configs.sh'] - custom: - command: [ 'sed', '-i', '$ aalias kafka-config.sh=/usr/bin/kafka-config', '/tmp/.bashrc'] - + command: [ 'chmod', '+x', '/tmp/kafka-topics.sh'] + - custom: + command: [ 'chmod', '+x', '/tmp/kafka-configs.sh'] production: apt: sources: - url: http://apt.wikimedia.org/wikimedia distribution: bullseye-wikimedia components: [main, thirdparty/confluent] packages: [ wmf-certificates, confluent-kafka-2.11, python3-pip ] copies: - from: datahub-source source: /datahub/docker/kafka-setup/kafka-setup.sh destination: ./kafka-setup.sh - from: datahub-source source: /datahub/docker/kafka-setup/kafka-config.sh destination: ./kafka-config.sh - from: datahub-source source: /datahub/docker/kafka-setup/kafka-topic-workers.sh destination: ./kafka-topic-workers.sh - from: dockerize source: /go/bin/dockerize destination: /usr/local/bin - from: cub source: /var/tmp/bin/cub destination: /usr/local/bin - from: cub source: /var/tmp/lib/python3.9 destination: /usr/local/lib/python3.9 - from : shell - source: /tmp/.bashrc - destination: /home/runuser/.bashrc + source: /tmp/kafka-topics.sh + destination: /usr/local/bin/kafka-topics.sh + - from : shell + source: /tmp/kafka-configs.sh + destination: /usr/local/bin/kafka-configs.sh runs: environment: + PATH: /datahub:$PATH PYTHONPATH: /usr/local/lib/python3.9/site-packages METADATA_AUDIT_EVENT_NAME: MetadataAuditEvent_v4 METADATA_CHANGE_EVENT_NAME: MetadataChangeEvent_v4 FAILED_METADATA_CHANGE_EVENT_NAME: FailedMetadataChangeEvent_v4 DATAHUB_USAGE_EVENT_NAME: DataHubUsageEvent_v1 METADATA_CHANGE_LOG_VERSIONED_TOPIC_NAME: MetadataChangeLog_Versioned_v1 METADATA_CHANGE_LOG_TIMESERIES_TOPIC_NAME: MetadataChangeLog_Timeseries_v1 METADATA_CHANGE_PROPOSAL_TOPIC_NAME: MetadataChangeProposal_v1 FAILED_METADATA_CHANGE_PROPOSAL_TOPIC_NAME: FailedMetadataChangeProposal_v1 PLATFORM_EVENT_TOPIC_NAME: PlatformEvent_v1 entrypoint: [ '/datahub/kafka-setup.sh']