commit 9b59107420ab527696b08ac89d787abd2c67d985 Author: Nicolas Vigier boklm@torproject.org Date: Wed Jul 19 18:33:12 2017 +0200

Build go-webrtc and snowflake for mac.

tor-browser-bundle.git author: David Fifield david@bamsoftware.com tor-browser-bundle.git commit: 26e0cd44f2886bfad1c3d30844ff7a21eb9d0478

Commit message from the tor-browser-bundle.git commit:

Build go-webrtc and snowflake in the mac pluggable-transports descriptor.

I had to apply two tricks to get a reproducible snowflake-client.

The first is to use faketime to eliminate some timestamps. There were 11 variable timestamps in the file. Through experimentation, I found that 10 of them were dependent on the Go runtime (recompiling Go caused them to change) and 1 was dependent on snowflake-client itself (recompiling snowflake-client with the same runtime changed only that 1 timestamp). The underlying issue has to do with clang 3.8.0 on Darwin embedding timestamps, unsolved in the Go issue tracker as of 13 days ago. https://github.com/golang/go/issues/9206#issuecomment-310476743

The second is a sed command to clobber embedded paths of the form /tmp/go-buildXXXXXXXXX and /tmp/go-link-XXXXXXXXX. Their presence is caused by some combination of Clang and Darwin, and there is as yet no known workaround upstream. --- projects/go-webrtc/config | 28 +++++++++++++++++++++------- projects/go/build | 7 ++++++- projects/go/config | 2 ++ projects/macosx-toolchain/build | 2 ++ projects/macosx-toolchain/config | 2 +- projects/snowflake/build | 36 +++++++++++++++++++++++++++++++++++- projects/snowflake/config | 4 ++++ rbm.conf | 5 ++++- 8 files changed, 75 insertions(+), 11 deletions(-)

diff --git a/projects/go-webrtc/config b/projects/go-webrtc/config index 673215b..c97a959 100644 --- a/projects/go-webrtc/config +++ b/projects/go-webrtc/config @@ -20,13 +20,23 @@ var: # Replace the prebuilt webrtc library with our own one. rm -rf include/ lib/ cp -a /var/tmp/dist/webrtc/{include,lib} . - # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with - # the prebuilt libwebrtc-magic.a libraries. Since we build our own - # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here, - # using CGO_CXXFLAGS. - # __STDC_FORMAT_MACROS is needed for a definition of PRIxPTR from inttypes.h. - export CGO_CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=1 -D__STDC_FORMAT_MACROS=1" - export CGO_LDFLAGS=-latomic + [% IF c("var/linux") -%] + # The go-webrtc package sets _GLIBCXX_USE_CXX11_ABI=0 for compatibility with + # the prebuilt libwebrtc-magic.a libraries. Since we build our own + # libwebrtc-magic.a with the C++11 ABI, we have to undo that setting here, + # using CGO_CXXFLAGS. + # __STDC_FORMAT_MACROS is needed for a definition of PRIxPTR from inttypes.h. + export CGO_CXXFLAGS="-D_GLIBCXX_USE_CXX11_ABI=1 -D__STDC_FORMAT_MACROS=1" + export CGO_LDFLAGS=-latomic + [% END -%] + [% IF c("var/osx") -%] + export CGO_ENABLED=1 + export CGO_CFLAGS="[% c("var/FLAGS") %] -mmacosx-version-min=10.7" + export CGO_CXXFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7" + export CGO_LDFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7" + export CC="$clangdir/bin/clang" + export CXX="$clangdir/bin/clang++" + [% END -%]

targets: master: @@ -36,6 +46,10 @@ targets: arch_deps: - pkg-config - libx11-dev + osx-x86_64: + var: + arch_deps: + - pkg-config

input_files: - project: container-image diff --git a/projects/go/build b/projects/go/build index 98f0522..3b51c32 100644 --- a/projects/go/build +++ b/projects/go/build @@ -52,7 +52,12 @@ cd /var/tmp/dist/go/src # http://golang.org/doc/install/source#environment export GOOS=darwin export GOARCH=amd64 - CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= ./make.bash + # faketime is needed because clang 3.8.0 on Darwin embeds the timestamps of + # certain intermediate object files (including those that arise here while + # compiling the Go runtime itself). Without this, variable timestamps would + # end up in snowflake-client. + # https://github.com/golang/go/issues/9206#issuecomment-310476743 + CGO_ENABLED=1 CC_FOR_TARGET="$CC_FOR_TARGET" CC= CFLAGS= LDFLAGS= [% c("var/faketime") %] ./make.bash [% ELSIF c("var/windows") -%] # Create a cc-for-target script that closes over CC, CFLAGS, and LDFLAGS. # Go's CC_FOR_TARGET only allows a command name, not a command with arguments. diff --git a/projects/go/config b/projects/go/config index 4b09e95..21669c9 100644 --- a/projects/go/config +++ b/projects/go/config @@ -60,6 +60,8 @@ targets: var: GOOS: darwin GOARCH: amd64 + arch_deps: + - faketime linux: var: GOOS: linux diff --git a/projects/macosx-toolchain/build b/projects/macosx-toolchain/build index e699e4f..f974cfc 100644 --- a/projects/macosx-toolchain/build +++ b/projects/macosx-toolchain/build @@ -10,6 +10,8 @@ tar -C $distdir -xf [% c('input_files_by_name/cctools') %] cd $distdir/cctools/bin ln -s ../../clang/bin/clang x86_64-apple-darwin10-clang ln -s ../../clang/bin/clang++ x86_64-apple-darwin10-clang++ +# "go link" expects to find a program called "dsymutil" exactly. +ln -sf x86_64-apple-darwin10-dsymutil dsymutil

cd "/var/tmp/dist" [% c('tar', { diff --git a/projects/macosx-toolchain/config b/projects/macosx-toolchain/config index 1779eb8..bfdbdb6 100644 --- a/projects/macosx-toolchain/config +++ b/projects/macosx-toolchain/config @@ -6,7 +6,7 @@ var: use_container: 1 setup: | mkdir -p /var/tmp/dist - tar -C /var/tmp/dist -xf [% c("compiler_tarfile") %] + tar -C /var/tmp/dist -xf $rootdir/[% c("compiler_tarfile") %] export sysrootdir=/var/tmp/dist/[% project %]/SDK/ export clangdir=/var/tmp/dist/[% project %]/clang export cctoolsdir=/var/tmp/dist/[% project %]/cctools/bin diff --git a/projects/snowflake/build b/projects/snowflake/build index 6e08b4e..9d9c1fb 100644 --- a/projects/snowflake/build +++ b/projects/snowflake/build @@ -10,6 +10,12 @@ distdir=/var/tmp/dist/[% project %] [% IF c("var/osx") %] PTDIR="$distdir/Contents/MacOS/Tor/PluggableTransports" DOCSDIR="$distdir/Contents/Resources/TorBrowser/Docs/snowflake" + export CGO_ENABLED=1 + export CGO_CFLAGS="[% c("var/FLAGS") %] -mmacosx-version-min=10.7" + export CGO_CXXFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7" + export CGO_LDFLAGS="[% c("var/FLAGS") %] -stdlib=libc++ -mmacosx-version-min=10.7" + export CC="$clangdir/bin/clang" + export CXX="$clangdir/bin/clang++" [% ELSE %] PTDIR="$distdir/TorBrowser/Tor/PluggableTransports" DOCSDIR="$distdir/TorBrowser/Docs/snowflake" @@ -25,7 +31,35 @@ tar -C /var/tmp/build -xf [% project %]-[% c('version') %].tar.gz cd /var/tmp/build/[% project %]-[% c('version') %]

cd client -go build -ldflags '-s' +[% IF c("var/osx") -%] + # Without faketime, snowflake-client would contain the timestamp of the + # temporary client.a file created during "go build". + [% c("var/faketime") %] go build -ldflags '-s' +[% ELSE -%] + go build -ldflags '-s' +[% END -%] +[% IF c("var/osx") -%] + # Hack: Overwrite variable absolute paths embedded in the binary. clang 3.8.0 + # on Darwin embeds such paths and the issue is unsolved in upstream Go as of + # 2016-06-28: + # https://github.com/golang/go/issues/9206#issuecomment-310476743 + # The two kinds of paths are ("000000000" stands for 9 random digits): + # /tmp/go-build000000000 + # /tmp/go-link-000000000 + # Such paths are the output of ioutil.TempDir("", "go-build") and + # ioutil.TempDir("", "go-link-"). + cp -a client client.stomped + sed -i -E -e 's#(/tmp/go-build|/tmp/go-link-)[0-9]{9}/#\1XXXXXXXXX/#g' client.stomped + # Sanity check: make sure the file actually changed. If it did not, it could + # mean that a change in go or clang has made this step unnecessary. + if cmp client client.stomped + then + echo "No paths replaced in snowflake-client. Maybe the replacement failed or is no longer needed." + echo "Check projects/snowflake/build" + exit 1 + fi + mv -f client.stomped client +[% END -%] cp -a client[% IF c("var/windows") %].exe[% END %] $PTDIR/snowflake-client[% IF c("var/windows") %].exe[% END %]

cd .. diff --git a/projects/snowflake/config b/projects/snowflake/config index 89e3eb6..4bf438e 100644 --- a/projects/snowflake/config +++ b/projects/snowflake/config @@ -14,6 +14,10 @@ targets: arch_deps: - pkg-config - libx11-dev + osx-x86_64: + var: + arch_deps: + - faketime

input_files: - project: container-image diff --git a/rbm.conf b/rbm.conf index c2143a9..49899ca 100644 --- a/rbm.conf +++ b/rbm.conf @@ -35,6 +35,7 @@ var: [% c("input_files_by_name/" _ file) %] [% END -%]

+ faketime: "faketime -f "[% USE date; GET date.format(c('timestamp'), format = '%Y-%m-%d %H:%M:%S') %]"" touch: "[% USE date %]touch -m -t [% date.format(c('timestamp'), format = '%Y%m%d%H%M') %]"

locale_ja: ja @@ -130,7 +131,7 @@ targets: var: linux: 1 compiler: gcc - # We only build snowflake on linux for now + # We only build snowflake on linux and osx for now snowflake: 1 container: suite: wheezy @@ -190,6 +191,8 @@ targets: FLAGS: "-target x86_64-apple-darwin10 -mlinker-version=136 -B $cctoolsdir -isysroot $sysrootdir" LDFLAGS: "-Wl,-syslibroot,$sysrootdir -Wl,-dead_strip -Wl,-pie" locale_ja: ja-JP-mac + # We only build snowflake on linux and osx for now + snowflake: 1 deps: - build-essential - python