commit 26e0cd44f2886bfad1c3d30844ff7a21eb9d0478 Author: David Fifield david@bamsoftware.com Date: Wed Jul 5 22:57:22 2017 -0700

Build go-webrtc and snowflake in the mac pluggable-transports descriptor.

I had to apply two tricks to get a reproducible snowflake-client.

The first is to use faketime to eliminate some timestamps. There were 11 variable timestamps in the file. Through experimentation, I found that 10 of them were dependent on the Go runtime (recompiling Go caused them to change) and 1 was dependent on snowflake-client itself (recompiling snowflake-client with the same runtime changed only that 1 timestamp). The underlying issue has to do with clang 3.8.0 on Darwin embedding timestamps, unsolved in the Go issue tracker as of 13 days ago. https://github.com/golang/go/issues/9206#issuecomment-310476743

The second is a sed command to clobber embedded paths of the form /tmp/go-buildXXXXXXXXX and /tmp/go-link-XXXXXXXXX. Their presence is caused by some combination of Clang and Darwin, and there is as yet no known workaround upstream. --- .../mac/gitian-pluggable-transports.yml | 71 +++++++++++++++++++++- gitian/mkbundle-mac.sh | 2 +- 2 files changed, 71 insertions(+), 2 deletions(-)

diff --git a/gitian/descriptors/mac/gitian-pluggable-transports.yml b/gitian/descriptors/mac/gitian-pluggable-transports.yml index 75ad899..ebfcaa9 100644 --- a/gitian/descriptors/mac/gitian-pluggable-transports.yml +++ b/gitian/descriptors/mac/gitian-pluggable-transports.yml @@ -6,7 +6,9 @@ suites: architectures: - "amd64" packages: +- "faketime" - "unzip" +- "pkg-config" - "zip" reference_datetime: "2000-01-01 00:00:00" remotes: @@ -24,6 +26,12 @@ remotes: "dir": "goxnet" - "url": "https://git.torproject.org/pluggable-transports/obfs4.git" "dir": "obfs4" +- "url": "https://github.com/keroserene/go-webrtc.git" + "dir": "go-webrtc" +- "url": "https://git.torproject.org/pluggable-transports/snowflake.git" + "dir": "snowflake" +- "url": "https://github.com/dchest/uniuri.git" + "dir": "uniuri" files: - "versions" - "go14.tar.gz" @@ -31,6 +39,7 @@ files: - "clang-linux64-jessie-utils.zip" - "cctools.tar.gz" - "MacOSX10.7.sdk.tar.gz" +- "webrtc-mac64-gbuilt.zip" - "dzip.sh" script: | INSTDIR="$HOME/install" @@ -59,6 +68,8 @@ script: | tar xaf MacOSX10.7.sdk.tar.gz # Preparing clang for cross-compilation, setting the proper flags and # variables + # "go link" expects to find a program called "dsymutil" exactly. + ln -sf x86_64-apple-darwin10-dsymutil $HOME/build/cctools/bin/dsymutil # ld needs libLTO.so from llvm export LD_LIBRARY_PATH="$HOME/build/clang/lib" export PATH="$HOME/build/cctools/bin:$PATH" @@ -93,7 +104,12 @@ script: | export GOARCH=amd64 tar xvf go.tar.gz cd go/src - CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= ./make.bash + # faketime is needed because clang 3.8.0 on Darwin embeds the timestamps of + # certain intermediate object files (including those that arise here while + # compiling the Go runtime itself). Without this, variable timestamps would + # end up in snowflake-client. + # https://github.com/golang/go/issues/9206#issuecomment-310476743 + CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= faketime -f "$REFERENCE_DATETIME" ./make.bash cd ../.. export PATH="$PATH:$PWD/go/bin"

@@ -138,6 +154,14 @@ script: | go install github.com/dchest/siphash cd ..

+ # Building go uniuri + cd uniuri + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/dchest/" + ln -sf "$PWD" "$GOPATH/src/github.com/dchest/uniuri" + go install github.com/dchest/uniuri + cd .. + # Building golang.org/x/crypto (obfs4proxy > 0.0.3 || Go >= 1.4) cd goxcrypto find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" @@ -166,6 +190,51 @@ script: | cp -a obfs4proxy $PTDIR cd ../..

+ unzip webrtc-mac64-gbuilt.zip + export SDKROOT="$PWD/MacOSX10.7.sdk" + + # Building go-webrtc + cd go-webrtc + # Replace the prebuilt webrtc library with our own one. + rm -rf include/ lib/ + ln -sf ../webrtc/{include,lib} . + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + mkdir -p "$GOPATH/src/github.com/keroserene/" + ln -sf "$PWD" "$GOPATH/src/github.com/keroserene/go-webrtc" + CFLAGS="$FLAGS -mmacosx-version-min=10.7" + CXXFLAGS="$FLAGS -stdlib=libc++ -mmacosx-version-min=10.7" + LDFLAGS="$FLAGS -stdlib=libc++ -mmacosx-version-min=10.7" + GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CFLAGS" CGO_CXXFLAGS="$CXXFLAGS" CGO_LDFLAGS="$LDFLAGS" CC="$HOME/build/clang/bin/clang" CXX="$HOME/build/clang/bin/clang++" go install github.com/keroserene/go-webrtc + cd .. + + # Building snowflake + cd snowflake + find -type f -print0 | xargs -0 touch --date="$REFERENCE_DATETIME" + cd client + # See the faketime comment above. Without faketime, snowflake-client would + # contain the timestamp of the temporary client.a file created during + # "go build". + GOARCH=amd64 CGO_ENABLED=1 CGO_CFLAGS="$CFLAGS" CGO_CXXFLAGS="$CXXFLAGS" CGO_LDFLAGS="$LDFLAGS" CC="$HOME/build/clang/bin/clang" CXX="$HOME/build/clang/bin/clang++" faketime -f "$REFERENCE_DATETIME" go build -ldflags '-s' + # Hack: Overwrite variable absolute paths embedded in the binary. clang 3.8.0 + # on Darwin embeds such paths and the issue is unsolved in upstream Go as of + # 2016-06-28: + # https://github.com/golang/go/issues/9206#issuecomment-310476743 + # The two kinds of paths are ("000000000" stands for 9 random digits): + # /tmp/go-build000000000 + # /tmp/go-link-000000000 + # Such paths are the output of ioutil.TempDir("", "go-build") and + # ioutil.TempDir("", "go-link-"). + cp -a client client.stomped + sed -i -E -e 's#(/tmp/go-build|/tmp/go-link-)[0-9]{9}/#\1XXXXXXXXX/#g' client.stomped + # Sanity check: make sure the file actually changed. If it did not, it could + # mean that a change in go or clang has made this step unnecessary. + cmp client client.stomped && (echo "No paths replaced in snowflake-client. Maybe the replacement failed or is no longer needed. Check descriptors/mac/gitian-pluggable-transports.yml"; exit 1) + cp -a client.stomped $PTDIR/snowflake-client + cd .. + mkdir -p $INSTDIR/Docs/snowflake + cp -a README.md LICENSE $INSTDIR/Docs/snowflake + cd .. + # Grabbing the result cd $INSTDIR ~/build/dzip.sh pluggable-transports-mac64-gbuilt.zip TorBrowserBundle.app diff --git a/gitian/mkbundle-mac.sh b/gitian/mkbundle-mac.sh index 646aca6..7bf546e 100755 --- a/gitian/mkbundle-mac.sh +++ b/gitian/mkbundle-mac.sh @@ -240,7 +240,7 @@ then echo "****** Starting Pluggable Transports Component of Mac Bundle (5/6 for Mac) ******" echo

- ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml + ./bin/gbuild -j $NUM_PROCS -m $VM_MEMORY --commit goptlib=$GOPTLIB_TAG,meek=$MEEK_TAG,ed25519=$GOED25519_TAG,siphash=$GOSIPHASH_TAG,goxcrypto=$GO_X_CRYPTO_TAG,goxnet=$GO_X_NET_TAG,obfs4=$OBFS4_TAG,go-webrtc=$GO_WEBRTC_TAG,snowflake=$SNOWFLAKE_TAG,uniuri=$UNIURI_TAG $DESCRIPTOR_DIR/mac/gitian-pluggable-transports.yml if [ $? -ne 0 ]; then #mv var/build.log ./firefox-fail-mac.log.`date +%Y%m%d%H%M%S`