airflow config update --fix, diff
ActivePublic
Actions

Authored by atsuko on Jul 29 2026, 1:10 PM.

Project Tags

None

Referenced Files

	F96397910: airflow config update --fix, diff
	Jul 29 2026, 1:10 PM

Subscribers

None

	--- airflow.cfg.bak 2026-07-29 13:07:51.228411633 +0000
	+++ airflow.cfg 2026-07-29 13:08:55.447606768 +0000
	@@ -1,95 +1,158 @@
	+[common.io]
	+# Common IO configuration section

	-[api]
	+# xcom_objectstorage_compression =

	-[common.io]
	-xcom_objectstorage_compression =
	xcom_objectstorage_path = s3://s3_dpe@logs.airflow-dev.dse-k8s-eqiad/dev-atsuko/xcoms
	xcom_objectstorage_threshold = 262144

	[core]
	auth_manager = airflow.providers.fab.auth_manager.fab_auth_manager.FabAuthManager
	colored_console_log = False
	-dags_are_paused_at_creation = True
	+# dags_are_paused_at_creation = True
	dags_folder = /opt/airflow/dags/airflow_dags/test_k8s
	default_task_retries = 5
	executor = KubernetesExecutor
	load_examples = False
	max_active_runs_per_dag = 3
	-max_active_tasks_per_dag = 16
	+# max_active_tasks_per_dag = 16
	max_map_length = 1200
	might_contain_dag_callable = import_helper.focused_import
	parallelism = 64
	test_connection = enabled
	-xcom_backend = airflow.providers.common.io.xcom.backend.XComObjectStorageBackend
	+xcom_backend = airflow.providers.common.io.xcom.backend.XComObjectStorageBackend
	security = kerberos

	[database]
	-load_default_connections = False
	+load_default_connections = False
	sql_alchemy_conn_cmd = /opt/airflow/usr/bin/pg_pooler_uri

	[datahub]
	cluster = prod
	conn_id = datahub_gms
	enabled = False

	[email]
	+# Configuration email backend and whether to
	+# send email alerts on retry or failure
	default_email_on_failure = False
	default_email_on_retry = False
	from_email = airflow-dev-atsuko <noreply@wikimedia.org>

	[kerberos]
	ccache = /tmp/airflow_krb5_ccache/krb5cc
	include_ip = False
	keytab = /etc/kerberos/keytabs/airflow.keytab
	principal = atsuko@WIKIMEDIA
	-reinit_frequency = 3600
	+# reinit_frequency = 3600

	[kubernetes_executor]
	delete_worker_pods_on_failure = True
	-multi_namespace_mode = False
	+# multi_namespace_mode = False
	namespace = airflow-dev
	pod_template_file = /opt/airflow/pod_templates/kubernetes_executor_default_pod_template.yaml
	worker_container_repository = docker-registry.discovery.wmnet/repos/data-engineering/airflow-dags
	worker_container_tag = airflow-3.3.0-py3.11-2026-07-29-110048-56a2261c888e8687c75ea9aaa4e2ab6dfdce33f8
	worker_pods_creation_batch_size = 32

	[logging]
	colored_console_log = False
	delete_local_logs = True
	-encrypt_s3_logs = False
	+# encrypt_s3_logs = False
	fab_logging_level = INFO
	-logging_level = INFO
	+# logging_level = INFO
	remote_base_log_folder = s3://logs.airflow-dev.dse-k8s-eqiad/dev-atsuko
	remote_log_conn_id = s3_dpe
	remote_logging = True

	[metrics]
	+# `StatsD <https://github.com/statsd/statsd>`__ integration settings.
	statsd_custom_client_path = wmf_airflow_common.metrics.custom_statsd_client.CustomStatsClient
	statsd_host = airflow-dev-atsuko-statsd
	statsd_on = True
	statsd_port = 9125
	-statsd_prefix = airflow
	+# statsd_prefix = airflow

	[scheduler]
	enable_health_check = True
	-max_tis_per_query = 16
	+# max_tis_per_query = 16
	min_file_process_interval = 300
	parsing_processes = 1
	pool_metrics_interval = 60
	-scheduler_health_check_server_host = 0.0.0.0
	+# scheduler_health_check_server_host = 0.0.0.0
	scheduler_health_check_server_port = 8765
	scheduler_health_check_threshold = 60
	standalone_dag_processor = False

	[secrets]
	backend = airflow.secrets.local_filesystem.LocalFilesystemBackend
	backend_kwargs = {"connections_file_path": "/opt/airflow/secrets/connections.yaml"}

	[smtp]
	+# If you want airflow to send emails on retries, failure, and you want to use
	+# the airflow.utils.email.send_email_smtp function, you have to configure an
	+# smtp server here
	smtp_host = mx-out1001.wikimedia.org
	smtp_mail_from = airflow-test-k8s: <noreply@wikimedia.org>
	-smtp_port = 25
	-smtp_ssl = False
	+# smtp_port = 25
	+# smtp_ssl = False
	smtp_starttls = False

	[triggerer]
	@@ -97,12 +160,22 @@

	[webserver]
	base_url = http://localhost:8080
	enable_proxy_fix = True
	expose_config = non-sensitive-only
	expose_hostname = True
	expose_stacktrace = True
	instance_name = dev-atsuko (airflow-test-k8s)
	rbac = True
	show_trigger_form_if_no_params = True
	warn_deployment_exposure = True
	web_server_worker_timeout = 300

Event Timeline

atsuko created this paste.Jul 29 2026, 1:10 PM

atsuko mentioned this in T433388: Adapt airflow chart for 3.3.x.

airflow config update --fix, diffActivePublicActions

Event Timeline

airflow config update --fix, diff
ActivePublic
Actions