We've had two reports of people failing to have their custom domains working and instead resulting in:
`` response from their domain.
This is symptomatic of the ingress that should have been created for them not existing.
Indeed we can see evidence of this in the logs:
```
{
"protoPayload": {
"@type": "type.googleapis.com/google.cloud.audit.AuditLog",
"authenticationInfo": {
"principalEmail": "system:serviceaccount:default:api-defaultrole"
},
"authorizationInfo": [
{
"granted": true,
"permission": "io.k8s.networking.v1.ingresses.create",
"resource": "networking.k8s.io/v1/namespaces/api-jobs/ingresses/mediawiki-site-589"
}
],
"methodName": "io.k8s.networking.v1.ingresses.create",
"request": {
"@type": "networking.k8s.io/v1.Ingress",
"apiVersion": "networking.k8s.io/v1",
"kind": "Ingress",
"metadata": {
"annotations": {
"cert-manager.io/cluster-issuer": "letsencrypt-prod",
"kubernetes.io/ingress.class": "nginx",
"nginx.ingress.kubernetes.io/force-ssl-redirect": "true"
},
"creationTimestamp": null,
"labels": {
"app.kubernetes.io/managed-by": "wbstack-platform",
"wbstack-ingress-generation": "2020-04-18.1",
"wbstack-wiki-domain": "christian-nationalists.org",
"wbstack-wiki-id": "589"
},
"name": "mediawiki-site-589",
"namespace": "default"
},
"spec": {
"rules": [
{
"host": "christian-nationalists.org",
"http": {
"paths": [
{
"backend": {
"service": {
"name": "platform-nginx",
"port": {
"number": 8080
}
}
},
"path": "/",
"pathType": "Prefix"
}
]
}
}
],
"tls": [
{
"hosts": [
"christian-nationalists.org"
],
"secretName": "mediawiki-site-tls-589"
}
]
},
"status": {
"loadBalancer": {}
}
},
"requestMetadata": {
"callerIp": "192.168.0.2",
"callerSuppliedUserAgent": "GuzzleHttp/6.5.5 curl/7.74.0 PHP/7.4.33"
},
"resourceName": "networking.k8s.io/v1/namespaces/api-jobs/ingresses/mediawiki-site-589",
"response": {
"@type": "core.k8s.io/v1.Status",
"apiVersion": "v1",
"code": 400,
"kind": "Status",
"message": "the namespace of the provided object does not match the namespace sent on the request",
"metadata": {},
"reason": "BadRequest",
"status": "Failure"
},
"serviceName": "k8s.io",
"status": {
"code": 3,
"message": "the namespace of the provided object does not match the namespace sent on the request"
}
},
"insertId": "947db941-12d4-4920-a1cf-0b0fd2c4c0e5",
"resource": {
"type": "k8s_cluster",
"labels": {
"location": "europe-west3-a",
"cluster_name": "wbaas-3",
"project_id": "wikibase-cloud"
}
},
"timestamp": "2023-08-31T18:57:49.670095Z",
"labels": {
"authorization.k8s.io/reason": "RBAC: allowed by ClusterRoleBinding \"api-defaultrole\" of ClusterRole \"api-defaultrole\" to ServiceAccount \"api-defaultrole/default\"",
"authorization.k8s.io/decision": "allow"
},
"logName": "projects/wikibase-cloud/logs/cloudaudit.googleapis.com%2Factivity",
"operation": {
"id": "947db941-12d4-4920-a1cf-0b0fd2c4c0e5",
"producer": "k8s.io",
"first": true,
"last": true
},
"receiveTimestamp": "2023-08-31T18:57:57.371393304Z"
}
```
This has probably happened since we moved api jobs to run in their own namespace.