mirror of
https://gitlab.torproject.org/tpo/anti-censorship/pluggable-transports/snowflake.git
synced 2025-10-13 11:11:30 -04:00
376 lines
11 KiB
YAML
376 lines
11 KiB
YAML
stages:
|
|
- test
|
|
- deploy
|
|
- container-build
|
|
|
|
variables:
|
|
DEBIAN_FRONTEND: noninteractive
|
|
DEBIAN_OLD_STABLE: buster
|
|
DEBIAN_STABLE: bullseye
|
|
REPRODUCIBLE_FLAGS: -trimpath -ldflags=-buildid=
|
|
|
|
# set up apt for automated use
|
|
.apt-template: &apt-template
|
|
- export LC_ALL=C.UTF-8
|
|
- export DEBIAN_FRONTEND=noninteractive
|
|
- echo Etc/UTC > /etc/timezone
|
|
- echo 'quiet "1";'
|
|
'APT::Install-Recommends "0";'
|
|
'APT::Install-Suggests "0";'
|
|
'APT::Acquire::Retries "20";'
|
|
'APT::Get::Assume-Yes "true";'
|
|
'Dpkg::Use-Pty "0";'
|
|
> /etc/apt/apt.conf.d/99gitlab
|
|
- apt-get update
|
|
- apt-get dist-upgrade
|
|
|
|
|
|
# Set things up to use the OS-native packages for Go. Anything that
|
|
# is downloaded by go during the `go fmt` stage is not coming from the
|
|
# Debian/Ubuntu repo. So those would need to be packaged for this to
|
|
# make it into Debian and/or Ubuntu.
|
|
.debian-native-template: &debian-native-template
|
|
variables:
|
|
GOPATH: /usr/share/gocode
|
|
before_script:
|
|
- apt-get update
|
|
- apt-get -qy install --no-install-recommends
|
|
build-essential
|
|
ca-certificates
|
|
git
|
|
golang
|
|
golang-github-cheekybits-genny-dev
|
|
golang-github-jtolds-gls-dev
|
|
golang-github-klauspost-reedsolomon-dev
|
|
golang-github-lucas-clemente-quic-go-dev
|
|
golang-github-smartystreets-assertions-dev
|
|
golang-github-smartystreets-goconvey-dev
|
|
golang-github-tjfoc-gmsm-dev
|
|
golang-github-xtaci-kcp-dev
|
|
golang-github-xtaci-smux-dev
|
|
golang-golang-x-crypto-dev
|
|
golang-golang-x-net-dev
|
|
golang-goptlib-dev
|
|
golang-golang-x-sys-dev
|
|
golang-golang-x-text-dev
|
|
golang-golang-x-xerrors-dev
|
|
lbzip2
|
|
|
|
# use Go installed as part of the official, Debian-based Docker images
|
|
.golang-docker-debian-template: &golang-docker-debian-template
|
|
before_script:
|
|
- apt-get update
|
|
- apt-get -qy install --no-install-recommends
|
|
ca-certificates
|
|
git
|
|
lbzip2
|
|
|
|
.go-test: &go-test
|
|
- gofmt -d .
|
|
- test -z "$(go fmt ./...)"
|
|
- go vet ./...
|
|
- go test -v -race ./...
|
|
|
|
- cd $CI_PROJECT_DIR/client/
|
|
- go get
|
|
- go build $REPRODUCIBLE_FLAGS
|
|
|
|
.test-template: &test-template
|
|
artifacts:
|
|
name: "${CI_PROJECT_PATH}_${CI_JOB_STAGE}_${CI_JOB_ID}_${CI_COMMIT_REF_NAME}_${CI_COMMIT_SHA}"
|
|
paths:
|
|
- client/*.aar
|
|
- client/*.jar
|
|
- client/client
|
|
expire_in: 1 week
|
|
when: on_success
|
|
after_script:
|
|
- echo "Download debug artifacts from https://gitlab.com/${CI_PROJECT_PATH}/-/jobs"
|
|
# this file changes every time but should not be cached
|
|
- rm -f $GRADLE_USER_HOME/caches/modules-2/modules-2.lock
|
|
- rm -rf $GRADLE_USER_HOME/caches/*/plugin-resolution/
|
|
|
|
# -- jobs ------------------------------------------------------------
|
|
|
|
android:
|
|
image: golang:1.21-$DEBIAN_STABLE
|
|
variables:
|
|
ANDROID_HOME: /usr/lib/android-sdk
|
|
LANG: C.UTF-8
|
|
cache:
|
|
paths:
|
|
- .gradle/wrapper
|
|
- .gradle/caches
|
|
<<: *test-template
|
|
before_script:
|
|
- *apt-template
|
|
- apt-get install
|
|
android-sdk-platform-23
|
|
android-sdk-platform-tools
|
|
build-essential
|
|
curl
|
|
default-jdk-headless
|
|
git
|
|
gnupg
|
|
unzip
|
|
wget
|
|
ca-certificates
|
|
lbzip2
|
|
|
|
- ndk=android-ndk-r21e-linux-x86_64.zip
|
|
- wget --continue --no-verbose https://dl.google.com/android/repository/$ndk
|
|
- echo "ad7ce5467e18d40050dc51b8e7affc3e635c85bd8c59be62de32352328ed467e $ndk" > $ndk.sha256
|
|
- sha256sum -c $ndk.sha256
|
|
- unzip -q $ndk
|
|
- rm ${ndk}*
|
|
- mv android-ndk-* $ANDROID_HOME/ndk-bundle/
|
|
|
|
- chmod -R a+rX $ANDROID_HOME
|
|
script:
|
|
- *go-test
|
|
- export GRADLE_USER_HOME=$CI_PROJECT_DIR/.gradle
|
|
- go version
|
|
- go env
|
|
|
|
- go get golang.org/x/mobile/cmd/gomobile
|
|
- go get golang.org/x/mobile/cmd/gobind
|
|
- go install golang.org/x/mobile/cmd/gobind
|
|
- go install golang.org/x/mobile/cmd/gomobile
|
|
- gomobile init
|
|
|
|
- cd $CI_PROJECT_DIR/client
|
|
# gomobile builds a shared library not a CLI executable
|
|
- sed -i 's,^package main$,package snowflakeclient,' *.go
|
|
- go get golang.org/x/mobile/bind
|
|
- gomobile bind -v -target=android $REPRODUCIBLE_FLAGS .
|
|
|
|
go-1.21:
|
|
image: golang:1.21-$DEBIAN_STABLE
|
|
<<: *golang-docker-debian-template
|
|
<<: *test-template
|
|
script:
|
|
- *go-test
|
|
|
|
debian-testing:
|
|
image: debian:testing
|
|
<<: *debian-native-template
|
|
<<: *test-template
|
|
script:
|
|
- *go-test
|
|
|
|
shadow-integration:
|
|
image: golang:1.21-$DEBIAN_STABLE
|
|
variables:
|
|
SHADOW_VERSION: "193924aae0dab30ffda0abe29467f552949849fa"
|
|
TGEN_VERSION: "v1.1.2"
|
|
cache:
|
|
key: sf-integration-$SHADOW_VERSION-$TGEN_VERSION
|
|
paths:
|
|
- /opt/
|
|
artifacts:
|
|
paths:
|
|
- shadow.data.tar.gz
|
|
when: on_failure
|
|
tags:
|
|
- amd64
|
|
- tpa
|
|
script:
|
|
- apt-get update
|
|
- apt-get install -y git tor
|
|
- mkdir -p ~/.local/bin
|
|
- mkdir -p ~/.local/src
|
|
- export PATH=$PATH:$CI_PROJECT_DIR/opt/bin/
|
|
|
|
# Install shadow and tgen
|
|
- pushd ~/.local/src
|
|
- |
|
|
if [ ! -f opt/shadow/bin/shadow ]
|
|
then
|
|
echo "The required version of shadow was not cached, building from source"
|
|
git clone --shallow-since=2021-08-01 https://github.com/shadow/shadow.git
|
|
pushd shadow/
|
|
git checkout $SHADOW_VERSION
|
|
CONTAINER=debian:stable-slim ci/container_scripts/install_deps.sh
|
|
CC=gcc CONTAINER=debian:stable-slim ci/container_scripts/install_extra_deps.sh
|
|
export PATH="$HOME/.cargo/bin:${PATH}"
|
|
./setup build --jobs $(nproc) --prefix $CI_PROJECT_DIR/opt/
|
|
./setup install
|
|
popd
|
|
fi
|
|
- |
|
|
if [ ! -f opt/shadow/bin/tgen ]
|
|
then
|
|
echo "The required version of tgen was not cached, building from source"
|
|
git clone --branch $TGEN_VERSION --depth 1 https://github.com/shadow/tgen.git
|
|
pushd tgen/
|
|
apt-get install -y cmake libglib2.0-dev libigraph-dev
|
|
mkdir build && cd build
|
|
cmake .. -DCMAKE_INSTALL_PREFIX=$CI_PROJECT_DIR/opt/
|
|
make
|
|
make install
|
|
popd
|
|
fi
|
|
install $CI_PROJECT_DIR/opt/bin/tgen ~/.local/bin/tgen
|
|
- popd
|
|
|
|
# Apply snowflake patch(es)
|
|
- |
|
|
git clone --depth 1 https://github.com/cohosh/shadow-snowflake-minimal
|
|
git am -3 shadow-snowflake-minimal/*.patch
|
|
|
|
# Install snowflake binaries to .local folder
|
|
- |
|
|
for app in "proxy" "client" "server" "broker" "probetest"; do
|
|
pushd $app
|
|
go build
|
|
install $app ~/.local/bin/snowflake-$app
|
|
popd
|
|
done
|
|
|
|
# Install stun server
|
|
- GOBIN=~/.local/bin go install github.com/gortc/stund@latest
|
|
|
|
# Run a minimal snowflake shadow experiment
|
|
- pushd shadow-snowflake-minimal/
|
|
- shadow --log-level=debug --model-unblocked-syscall-latency=true snowflake-minimal.yaml > shadow.log
|
|
- tar -czvf $CI_PROJECT_DIR/shadow.data.tar.gz shadow.data/
|
|
|
|
# Check to make sure streams succeeded
|
|
- |
|
|
if [ $(grep -c "stream-success" shadow.data/hosts/snowflakeclient/tgen.*.stdout) = 10 ]
|
|
then
|
|
echo "All streams in shadow completed successfully"
|
|
else
|
|
echo "Shadow simulation failed"
|
|
exit 1
|
|
fi
|
|
|
|
generate_tarball:
|
|
stage: deploy
|
|
image: golang:1.21-$DEBIAN_STABLE
|
|
rules:
|
|
- if: $CI_COMMIT_TAG
|
|
script:
|
|
- go mod vendor
|
|
- tar czf ${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz --transform "s,^,${CI_PROJECT_NAME}-${CI_COMMIT_TAG}/," *
|
|
after_script:
|
|
- echo TAR_JOB_ID=$CI_JOB_ID >> generate_tarball.env
|
|
artifacts:
|
|
paths:
|
|
- ${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz
|
|
reports:
|
|
dotenv: generate_tarball.env
|
|
|
|
release-job:
|
|
stage: deploy
|
|
image: registry.gitlab.com/gitlab-org/release-cli:latest
|
|
rules:
|
|
- if: $CI_COMMIT_TAG
|
|
needs:
|
|
- job: generate_tarball
|
|
artifacts: true
|
|
script:
|
|
- echo "running release_job"
|
|
release:
|
|
name: 'Release $CI_COMMIT_TAG'
|
|
description: 'Created using the release-cli'
|
|
tag_name: '$CI_COMMIT_TAG'
|
|
ref: '$CI_COMMIT_TAG'
|
|
assets:
|
|
links:
|
|
- name: '${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz'
|
|
url: '${CI_PROJECT_URL}/-/jobs/${TAR_JOB_ID}/artifacts/file/${CI_PROJECT_NAME}-${CI_COMMIT_TAG}.tar.gz'
|
|
|
|
# Build the container only if the commit is to main, or it is a tag.
|
|
# If the commit is to main, then the docker image tag should be set to `latest`.
|
|
# If it is a tag, then the docker image tag should be set to the tag name.
|
|
build-container:
|
|
variables:
|
|
TAG: $CI_COMMIT_TAG # Will not be set on a non-tag build, will be set later
|
|
stage: container-build
|
|
parallel:
|
|
matrix:
|
|
- ARCH: amd64
|
|
- ARCH: arm64
|
|
- ARCH: s390x
|
|
tags:
|
|
- $ARCH
|
|
image:
|
|
name: gcr.io/kaniko-project/executor:debug
|
|
entrypoint: [""]
|
|
script:
|
|
- if [ $CI_COMMIT_REF_NAME == "main" ]; then export TAG='latest'; fi
|
|
- >-
|
|
echo "Building Docker image with tag: $TAG"
|
|
/kaniko/executor
|
|
--context "${CI_PROJECT_DIR}"
|
|
--dockerfile "${CI_PROJECT_DIR}/Dockerfile"
|
|
--destination "${CI_REGISTRY_IMAGE}:${TAG}_${ARCH}"
|
|
rules:
|
|
- if: $CI_COMMIT_REF_NAME == "main"
|
|
- if: $CI_COMMIT_TAG
|
|
|
|
merge-manifests:
|
|
variables:
|
|
TAG: $CI_COMMIT_TAG
|
|
stage: container-build
|
|
needs:
|
|
- job: build-container
|
|
artifacts: false
|
|
image:
|
|
name: mplatform/manifest-tool:alpine
|
|
entrypoint: [""]
|
|
script:
|
|
- if [ $CI_COMMIT_REF_NAME == "main" ]; then export TAG='latest'; fi
|
|
- >-
|
|
manifest-tool
|
|
--username="${CI_REGISTRY_USER}"
|
|
--password="${CI_REGISTRY_PASSWORD}"
|
|
push from-args
|
|
--platforms linux/amd64,linux/arm64,linux/s390x
|
|
--template "${CI_REGISTRY_IMAGE}:${TAG}_ARCH"
|
|
--target "${CI_REGISTRY_IMAGE}:${TAG}"
|
|
rules:
|
|
- if: $CI_COMMIT_REF_NAME == "main"
|
|
when: always
|
|
- if: $CI_COMMIT_TAG
|
|
when: always
|
|
|
|
# If this is a tag, then we want to additionally tag the image as `stable`
|
|
tag-container-release:
|
|
stage: container-build
|
|
needs:
|
|
- job: merge-manifests
|
|
artifacts: false
|
|
image: quay.io/podman/stable
|
|
allow_failure: false
|
|
variables:
|
|
IMAGE_TAG: $CI_REGISTRY_IMAGE:$CI_COMMIT_TAG
|
|
RELEASE_TAG: $CI_REGISTRY_IMAGE:stable
|
|
script:
|
|
- echo "Tagging docker image with stable tag"
|
|
- echo -n "$CI_JOB_TOKEN" | podman login -u gitlab-ci-token --password-stdin $CI_REGISTRY
|
|
- podman pull $IMAGE_TAG || true
|
|
- podman tag $IMAGE_TAG $RELEASE_TAG
|
|
- podman push $RELEASE_TAG
|
|
rules:
|
|
- if: $CI_COMMIT_TAG
|
|
when: always
|
|
|
|
clean-image-tags:
|
|
stage: container-build
|
|
needs:
|
|
- job: merge-manifests
|
|
artifacts: false
|
|
image: containers.torproject.org/tpo/tpa/base-images:bookworm
|
|
before_script:
|
|
- *apt-template
|
|
- apt-get install -y jq curl
|
|
script:
|
|
- "REGISTRY_ID=$(curl --silent --request GET --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" \"https://gitlab.torproject.org/api/v4/projects/${CI_PROJECT_ID}/registry/repositories\" | jq '.[].id')"
|
|
- "curl --request DELETE --data \"name_regex_delete=(latest|${CI_COMMIT_TAG})_.*\" --header \"JOB-TOKEN: ${CI_JOB_TOKEN}\" \"https://gitlab.torproject.org/api/v4/projects/${CI_PROJECT_ID}/registry/repositories/${REGISTRY_ID}/tags\""
|
|
rules:
|
|
- if: $CI_COMMIT_REF_NAME == "main"
|
|
when: always
|
|
- if: $CI_COMMIT_TAG
|
|
when: always
|