Mike Hommey
f2685d7dd2
In some cases, IPC messages between the sccache client and sccache server are too large for the default frame length, and we had a work around for some of those cases that happened in CI some time ago. Turning on the sccache client-side mode makes this IPC default frame length problem worse, preventing the caching of many rust crate build output. So at least until sccache is improved to handle the situation more smoothly, we're going to need to increase the frame length. The previous workaround only applied to automation, where mozconfig.cache is used. We however now need to apply it more globally, setting the variable from configure whenever sccache is in use instead, alongside the other sccache variables, which also gets it in the environment of whatever starts the sccache daemon: configure's own compiler checks, make, or client.mk. This only moves the setting, without changing the value just yet. Differential Revision: https://phabricator.services.mozilla.com/D316436
94 lines
4.6 KiB
Plaintext
94 lines
4.6 KiB
Plaintext
# This Source Code Form is subject to the terms of the Mozilla Public
|
|
# License, v. 2.0. If a copy of the MPL was not distributed with this
|
|
# file, You can obtain one at http://mozilla.org/MPL/2.0/.
|
|
|
|
# Setup for build cache
|
|
|
|
# Thunderbird builds will set prior to including this file
|
|
aws_prefix=${aws_prefix:-taskcluster}
|
|
gcp_prefix=${gcp_prefix:-sccache}
|
|
|
|
if test -n "$SCCACHE_DISABLE" -a -n "$USE_SCCACHE"; then
|
|
echo "SetupError: SCCACHE_DISABLE and USE_SCCACHE should not be set at the same time" >&2
|
|
exit 1
|
|
fi
|
|
|
|
# builds where buildprops didn't have the data (eg: taskcluster) and without sccache disabled:
|
|
if test -z "$bucket" -a -z "$SCCACHE_DISABLE"; then
|
|
|
|
# prevent rerun if az is set, or wget is not available
|
|
if test -z "$availability_zone" -a -x "$(command -v wget)"; then
|
|
if test -n "$TASKCLUSTER_WORKER_LOCATION" -a -x "$(command -v jq)"; then
|
|
cloud=$(echo $TASKCLUSTER_WORKER_LOCATION | jq .cloud | tr -d \")
|
|
case $cloud in
|
|
aws|google)
|
|
availability_zone=$(echo $TASKCLUSTER_WORKER_LOCATION | jq .availabilityZone | tr -d \")
|
|
region=$(echo $TASKCLUSTER_WORKER_LOCATION | jq .region | tr -d \")
|
|
;;
|
|
esac
|
|
fi
|
|
if test -z "$availability_zone" -o -z "$region"; then
|
|
if test -n "${TASKCLUSTER_WORKER_GROUP}"; then
|
|
# TASKCLUSTER_WORKER_GROUP is just the region now, so
|
|
# stick an extra character on to make the already-convoluted logic
|
|
# here simpler.
|
|
availability_zone="${TASKCLUSTER_WORKER_GROUP}x"
|
|
elif [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
|
|
# gcp availability_zone is of the form <region>-<letter> where region is e.g. us-west2, and az is us-west2-a
|
|
gcp_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/computeMetadata/v1beta1/instance/zone || true)
|
|
availability_zone=${gcp_zone##*/}
|
|
else
|
|
# timeout after 1 second, and don't retry (failure indicates instance is not in ec2 or network issue)
|
|
# ec2 availability_zone is of the form <region><letter> where region is e.g. us-west-2, and az is us-west-2a
|
|
availability_zone=$(wget -T 1 -t 1 -q -O - http://169.254.169.254/latest/meta-data/placement/availability-zone || true)
|
|
fi
|
|
fi
|
|
if test -z "$availability_zone" -o "$availability_zone" = "not-ec2"; then
|
|
availability_zone=not-ec2
|
|
elif test -n "$cloud"; then
|
|
case $cloud in
|
|
google)
|
|
bucket=${gcp_prefix}-l${MOZ_SCM_LEVEL}-${region}
|
|
# deal with length limit on serviceaccount names (https://bugzilla.mozilla.org/show_bug.cgi?id=1882374#c5)
|
|
serviceaccount=$(echo $bucket | sed s/northamerica/na/)
|
|
;;
|
|
aws)
|
|
bucket=${aws_prefix}-level-${MOZ_SCM_LEVEL}-sccache-${region}
|
|
;;
|
|
esac
|
|
elif [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
|
|
# gcp region is az with last two letters trimmed
|
|
if test -z "$region"; then
|
|
region=${availability_zone::${#availability_zone}-2}
|
|
fi
|
|
bucket=${aws_prefix}-level-${MOZ_SCM_LEVEL}-sccache-${region}
|
|
else
|
|
# ec2 region is az with last letter trimmed
|
|
if test -z "$region"; then
|
|
region=${availability_zone%?}
|
|
fi
|
|
bucket=${aws_prefix}-level-${MOZ_SCM_LEVEL}-sccache-${region}
|
|
fi
|
|
fi
|
|
fi
|
|
|
|
if test -n "$bucket"; then
|
|
if test "$cloud" = "google"; then
|
|
mk_add_options "export SCCACHE_GCS_BUCKET=$bucket"
|
|
mk_add_options "export SCCACHE_GCS_RW_MODE=READ_WRITE"
|
|
mk_add_options "export SCCACHE_GCS_CREDENTIALS_URL=http://taskcluster/auth/v1/gcp/credentials/$SCCACHE_GCS_PROJECT/${serviceaccount}@$SCCACHE_GCS_PROJECT.iam.gserviceaccount.com"
|
|
elif [ -n "${SCCACHE_GCS_KEY_PATH}" ]; then
|
|
mk_add_options "export SCCACHE_GCS_BUCKET=$bucket"
|
|
else
|
|
mk_add_options "export SCCACHE_BUCKET=$bucket"
|
|
# instruct sccache to fetch the credentials from the Auth service's awsS3Credentials endpoint, via the Taskcluster proxy.
|
|
mk_add_options "export AWS_IAM_CREDENTIALS_URL=http://taskcluster/auth/v1/aws/s3/read-write/${bucket}/?format=iam-role-compat"
|
|
fi
|
|
export CCACHE="sccache"
|
|
export SCCACHE_VERBOSE_STATS=1
|
|
mk_add_options MOZBUILD_MANAGE_SCCACHE_DAEMON=${MOZ_FETCHES_DIR}/sccache/sccache
|
|
elif test -n "$USE_SCCACHE"; then
|
|
echo "SetupError: sccache is enabled but there is no shared storage set up for it?" >&2
|
|
exit 1
|
|
fi
|