Skip to content

Commit e243406

Browse files
kripkenkripken
authored
Prepare for emscripten-releases (#237)
Updated to our current initial plan here: * Add emscripten-releases-tags.txt which is a JSON file with a map of tag name to git hash in the emscripten-releases repo. Right now this file is just checked in here; in the future we may make the emsdk update it from git tags with update-tags etc. * That file contains a "latest" tag, which is fetched for when getting latest-releases-[upstream|fastcomp]. We may want that to eventually be updated based on the latest green stuff on the bots perhaps. * Rename the current sdk-[upstream|fastcomp]-* etc. to sdk-waterfall-[upstream\|fastcomp]-*. Then we'll have sdk-releases-[upstream|fastcomp]-* etc. This is just an internal name change - emsdk install latest-upstream still works as before, and still uses the waterfall (so we don't break our github CI). * Add support for emsdk install latest-releases-[upstream\|fastcomp] * Add a test using the actual emscripten-releases builds
1 parent 751355a commit e243406

File tree

4 files changed

+124
-27
lines changed

4 files changed

+124
-27
lines changed

‎Dockerfile

Lines changed: 12 additions & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -19,7 +19,7 @@ RUN cd /root/ \
1919
&& /root/emsdk/emsdk install latest-upstream \
2020
&& /root/emsdk/emsdk activate latest-upstream \
2121
&& source /root/emsdk/emsdk_env.sh --build=Release \
22-
&& emcc hello_world.cpp -s WASM_OBJECT_FILES=1 \
22+
&& emcc hello_world.cpp \
2323
&& python -c "import os ; assert open(os.path.expanduser('~/.emscripten')).read().count('LLVM_ROOT') == 1" \
2424
&& python -c "import os ; assert 'upstream' in open(os.path.expanduser('~/.emscripten')).read()" \
2525
&& echo "test fastcomp (waterfall)" \
@@ -28,5 +28,16 @@ RUN cd /root/ \
2828
&& source /root/emsdk/emsdk_env.sh --build=Release \
2929
&& emcc hello_world.cpp \
3030
&& emcc hello_world.cpp -s WASM=0 \
31+
&& emcc --clear-cache \
32+
&& echo "test latest-releases-upstream" \
33+
&& /root/emsdk/emsdk install latest-releases-upstream \
34+
&& /root/emsdk/emsdk activate latest-releases-upstream \
35+
&& source /root/emsdk/emsdk_env.sh --build=Release \
36+
&& emcc hello_world.cpp \
37+
&& echo "test latest-releases-fastcomp" \
38+
&& /root/emsdk/emsdk install latest-releases-fastcomp \
39+
&& /root/emsdk/emsdk activate latest-releases-fastcomp \
40+
&& source /root/emsdk/emsdk_env.sh --build=Release \
41+
&& emcc hello_world.cpp \
3142
&& echo "test binaryen source build" \
3243
&& /root/emsdk/emsdk install --build=Release binaryen-master-64bit

‎emscripten-releases-tags.txt

Lines changed: 4 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,4 @@
1+
{
2+
"latest": "0ea08ee8a5f0524b4c9ac94de4f161ceeacb3e74"
3+
}
4+

‎emsdk

Lines changed: 44 additions & 24 deletions
Original file line numberDiff line numberDiff line change
@@ -88,6 +88,14 @@ ENABLE_WASM = False
8888
ENABLE_LLVM_ASSERTIONS = 'auto'
8989

9090

91+
def os_name():
92+
if WINDOWS: return 'win'
93+
elif LINUX: return 'linux'
94+
elif OSX: return 'osx'
95+
else:
96+
raise Exception('unknown OS')
97+
98+
9199
def to_unix_path(p):
92100
return p.replace('\\', '/')
93101

@@ -1653,15 +1661,12 @@ def find_latest_waterfall_sdk(which):
16531661
if not waterfall_lkgr:
16541662
print('Failed to find an upstream lkgr')
16551663
sys.exit(1)
1656-
return 'sdk-%s-%s-64bit' % (which, waterfall_lkgr[0])
1657-
1664+
return 'sdk-waterfall-%s-%s-64bit' % (which, waterfall_lkgr[0])
16581665

1659-
def find_latest_upstream_sdk():
1660-
return find_latest_waterfall_sdk('upstream')
16611666

1662-
1663-
def find_latest_fastcomp_sdk():
1664-
return find_latest_waterfall_sdk('fastcomp')
1667+
def find_latest_releases_sdk(which):
1668+
releases_info = load_releases_info()
1669+
return 'sdk-releases-%s-%s-64bit' % (which, releases_info['latest'])
16651670

16661671

16671672
# Finds the best-matching python tool for use.
@@ -1732,26 +1737,17 @@ def fetch_emscripten_tags():
17321737
else:
17331738
print('Done. No tagged Binaryen releases available.')
17341739

1735-
def os_name_for_llvm_location():
1736-
if WINDOWS: return 'win'
1737-
if LINUX: return 'linux'
1738-
if OSX: return 'osx'
1739-
1740-
def os_name_for_emscripten_location():
1741-
if WINDOWS: return 'win'
1742-
else: return 'linux'
1743-
17441740
# Emscripten Nightlies support has been removed, clear the list of known Nightlies locally.
17451741
# print('Fetching all precompiled Nightly versions..')
1746-
# download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
1747-
# download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
1748-
# download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name_for_emscripten_location() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
1742+
# download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name() + '_32bit/index.txt', 'llvm-nightlies-32bit.txt', download_even_if_exists=True)
1743+
# download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/nightly/' + os_name() + '_64bit/index.txt', 'llvm-nightlies-64bit.txt', download_even_if_exists=True)
1744+
# download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/emscripten/nightly/' + os_name() + '/index.txt', 'emscripten-nightlies.txt', download_even_if_exists=True)
17491745
for f in ['llvm-nightlies-32bit.txt', 'llvm-nightlies-64bit.txt', 'emscripten-nightlies.txt']:
17501746
if os.path.isfile(f): os.remove(f)
17511747

17521748
print('Fetching all precompiled tagged releases..')
1753-
download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
1754-
download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name_for_llvm_location() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
1749+
download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name() + '_32bit/index.txt', 'llvm-tags-32bit.txt', download_even_if_exists=True)
1750+
download_file('https://s3.amazonaws.com/mozilla-games/emscripten/packages/llvm/tag/' + os_name() + '_64bit/index.txt', 'llvm-tags-64bit.txt', download_even_if_exists=True)
17551751
download_waterfall_lkgr()
17561752

17571753
if not git:
@@ -1849,7 +1845,7 @@ def load_llvm_precompiled_tags_64bit():
18491845

18501846

18511847
def download_waterfall_lkgr():
1852-
lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/linux/lkgr.json'
1848+
lkgr_url = 'https://storage.googleapis.com/wasm-llvm/builds/%s/lkgr.json' % os_name()
18531849
download_file(lkgr_url, 'upstream', download_even_if_exists=True)
18541850

18551851

@@ -1865,6 +1861,23 @@ def load_waterfall_lkgr():
18651861
return []
18661862

18671863

1864+
# Load the json info for emscripten-releases.
1865+
def load_releases_info():
1866+
try:
1867+
text = open(sdk_path('emscripten-releases-tags.txt'), 'r').read()
1868+
return json.loads(text)
1869+
except Exception as e:
1870+
print('Error parsing emscripten-releases-tags.txt!')
1871+
print(str(e))
1872+
sys.exit(1)
1873+
1874+
1875+
# Get a list of tags for emscripten-releases.
1876+
def load_releases_tags():
1877+
info = load_releases_info()
1878+
return info.values()
1879+
1880+
18681881
def is_string(s):
18691882
if sys.version_info[0] >= 3:
18701883
return isinstance(s, str)
@@ -1889,6 +1902,7 @@ def load_sdk_manifest():
18891902
llvm_64bit_nightlies = list(reversed(load_llvm_64bit_nightlies()))
18901903
emscripten_nightlies = list(reversed(load_emscripten_nightlies()))
18911904
waterfall_lkgr = load_waterfall_lkgr()
1905+
releases_tags = load_releases_tags()
18921906

18931907
def dependencies_exist(sdk):
18941908
for tool_name in sdk.uses:
@@ -1960,6 +1974,7 @@ def load_sdk_manifest():
19601974
elif '%nightly-llvm-32bit%' in t.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, t, is_sdk=False)
19611975
elif '%nightly-emscripten%' in t.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, t, is_sdk=False)
19621976
elif '%waterfall-lkgr%' in t.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, t, is_sdk=False)
1977+
elif '%releases-tag%' in t.version: expand_category_param('%releases-tag%', releases_tags, t, is_sdk=False)
19631978
else:
19641979
add_tool(t)
19651980

@@ -1978,6 +1993,7 @@ def load_sdk_manifest():
19781993
elif '%nightly-llvm-32bit%' in sdk.version: expand_category_param('%nightly-llvm-32bit%', llvm_32bit_nightlies, sdk, is_sdk=True)
19791994
elif '%nightly-emscripten%' in sdk.version: expand_category_param('%nightly-emscripten%', emscripten_nightlies, sdk, is_sdk=True)
19801995
elif '%waterfall-lkgr%' in sdk.version: expand_category_param('%waterfall-lkgr%', waterfall_lkgr, sdk, is_sdk=True)
1996+
elif '%releases-tag%' in sdk.version: expand_category_param('%releases-tag%', releases_tags, sdk, is_sdk=True)
19811997
else:
19821998
add_sdk(sdk)
19831999

@@ -2388,9 +2404,13 @@ def main():
23882404
elif sys.argv[i] == 'sdk-nightly-latest-64bit':
23892405
sys.argv[i] = str(find_latest_nightly_64bit_sdk())
23902406
elif sys.argv[i] == 'latest-upstream' or sys.argv[i] == 'latest-clang-upstream':
2391-
sys.argv[i] = str(find_latest_upstream_sdk())
2407+
sys.argv[i] = str(find_latest_waterfall_sdk('upstream'))
23922408
elif sys.argv[i] == 'latest-fastcomp':
2393-
sys.argv[i] = str(find_latest_fastcomp_sdk())
2409+
sys.argv[i] = str(find_latest_waterfall_sdk('fastcomp'))
2410+
elif sys.argv[i] == 'latest-releases-upstream':
2411+
sys.argv[i] = str(find_latest_releases_sdk('upstream'))
2412+
elif sys.argv[i] == 'latest-releases-fastcomp':
2413+
sys.argv[i] = str(find_latest_releases_sdk('fastcomp'))
23942414

23952415
if cmd == 'list':
23962416
print('')

‎emsdk_manifest.json

Lines changed: 64 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -210,6 +210,32 @@
210210
"activated_path": "%installation_dir%/emscripten",
211211
"activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'"
212212
},
213+
214+
{
215+
"id": "releases",
216+
"version": "upstream-%releases-tag%",
217+
"bitness": 64,
218+
"linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
219+
"osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
220+
"windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/windows/%releases-tag%/wasm-binaries.tbz2",
221+
"zipfile_prefix": "%releases-tag%-",
222+
"install_path": "upstream/%releases-tag%",
223+
"activated_path": "%installation_dir%/emscripten",
224+
"activated_cfg": "LLVM_ROOT='%installation_dir%/bin';BINARYEN_ROOT='%installation_dir%'"
225+
},
226+
{
227+
"id": "releases",
228+
"version": "fastcomp-%releases-tag%",
229+
"bitness": 64,
230+
"linux_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/linux/%releases-tag%/wasm-binaries.tbz2",
231+
"osx_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/mac/%releases-tag%/wasm-binaries.tbz2",
232+
"windows_url": "https://storage.googleapis.com/webassembly/emscripten-releases-builds/windows/%releases-tag%/wasm-binaries.zip",
233+
"zipfile_prefix": "%releases-tag%-",
234+
"install_path": "fastcomp/%releases-tag%",
235+
"activated_path": "%installation_dir%/emscripten",
236+
"activated_cfg": "LLVM_ROOT='%installation_dir%/fastcomp/bin';BINARYEN_ROOT='%installation_dir%'"
237+
},
238+
213239
{
214240
"id": "clang",
215241
"version": "e1.13.0",
@@ -1585,17 +1611,53 @@
15851611
"os": "win"
15861612
},
15871613
{
1588-
"version": "upstream-%waterfall-lkgr%",
1614+
"version": "waterfall-upstream-%waterfall-lkgr%",
15891615
"bitness": 64,
15901616
"uses": ["waterfall-upstream-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
15911617
"os": "linux"
15921618
},
15931619
{
1594-
"version": "fastcomp-%waterfall-lkgr%",
1620+
"version": "waterfall-fastcomp-%waterfall-lkgr%",
15951621
"bitness": 64,
15961622
"uses": ["waterfall-fastcomp-%waterfall-lkgr%-64bit", "node-8.9.1-64bit"],
15971623
"os": "linux"
15981624
},
1625+
{
1626+
"version": "releases-upstream-%releases-tag%",
1627+
"bitness": 64,
1628+
"uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
1629+
"os": "linux"
1630+
},
1631+
{
1632+
"version": "releases-upstream-%releases-tag%",
1633+
"bitness": 64,
1634+
"uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
1635+
"os": "osx"
1636+
},
1637+
{
1638+
"version": "releases-upstream-%releases-tag%",
1639+
"bitness": 64,
1640+
"uses": ["releases-upstream-%releases-tag%-64bit", "node-8.9.1-64bit"],
1641+
"os": "win"
1642+
},
1643+
{
1644+
"version": "releases-fastcomp-%releases-tag%",
1645+
"bitness": 64,
1646+
"uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
1647+
"os": "linux"
1648+
},
1649+
{
1650+
"version": "releases-fastcomp-%releases-tag%",
1651+
"bitness": 64,
1652+
"uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
1653+
"os": "osx"
1654+
},
1655+
{
1656+
"version": "releases-fastcomp-%releases-tag%",
1657+
"bitness": 64,
1658+
"uses": ["releases-fastcomp-%releases-tag%-64bit", "node-8.9.1-64bit"],
1659+
"os": "win"
1660+
},
15991661
{
16001662
"version": "%precompiled_tag32%",
16011663
"bitness": 32,

0 commit comments

Comments
 (0)