Skip to content

Commit d9aed28

Browse files
authored
Move some code out of tools/building.py. NFC (#21782)
Some of these function are only called in single file so there is no need for them to be in this shared location.
1 parent cfc3ce9 commit d9aed28

File tree

3 files changed

+127
-128
lines changed

3 files changed

+127
-128
lines changed

test/test_other.py

Lines changed: 27 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -58,6 +58,24 @@
5858
wasm_opt = Path(building.get_binaryen_bin(), 'wasm-opt')
5959

6060

61+
def is_bitcode(filename):
62+
try:
63+
# look for magic signature
64+
b = open(filename, 'rb').read(4)
65+
if b[:2] == b'BC':
66+
return True
67+
# on macOS, there is a 20-byte prefix which starts with little endian
68+
# encoding of 0x0B17C0DE
69+
elif b == b'\xDE\xC0\x17\x0B':
70+
b = bytearray(open(filename, 'rb').read(22))
71+
return b[20:] == b'BC'
72+
except IndexError:
73+
# not enough characters in the input
74+
# note that logging will be done on the caller function
75+
pass
76+
return False
77+
78+
6179
def uses_canonical_tmp(func):
6280
"""Decorator that signals the use of the canonical temp by a test method.
6381

@@ -5048,7 +5066,7 @@ def test_bitcode_linking(self):
50485066
self.run_process([EMCC, '-flto', '-c', test_file('hello_world.c')])
50495067
self.assertExists('hello_world.o')
50505068
self.run_process([EMCC, '-flto', '-r', 'hello_world.o', '-o', 'hello_world2.o'])
5051-
building.is_bitcode('hello_world.o')
5069+
is_bitcode('hello_world.o')
50525070
building.is_wasm('hello_world2.o')
50535071

50545072
@parameterized({
@@ -5588,7 +5606,7 @@ def test_emit_llvm_asm(self):
55885606

55895607
def test_emit_llvm(self):
55905608
self.run_process([EMCC, test_file('hello_world.c'), '-c', '-emit-llvm'])
5591-
self.assertTrue(building.is_bitcode('hello_world.bc'))
5609+
self.assertTrue(is_bitcode('hello_world.bc'))
55925610

55935611
def test_compile_ll_file(self):
55945612
self.run_process([EMCC, test_file('hello_world.c'), '-S', '-emit-llvm'])
@@ -8775,12 +8793,12 @@ def test_lto(self):
87758793
print('wasm in object')
87768794
self.run_process([EMXX, src] + args + ['-c', '-o', 'hello_obj.o'])
87778795
self.assertTrue(building.is_wasm('hello_obj.o'))
8778-
self.assertFalse(building.is_bitcode('hello_obj.o'))
8796+
self.assertFalse(is_bitcode('hello_obj.o'))
87798797

87808798
print('bitcode in object')
87818799
self.run_process([EMXX, src] + args + ['-c', '-o', 'hello_bitcode.o', '-flto'])
87828800
self.assertFalse(building.is_wasm('hello_bitcode.o'))
8783-
self.assertTrue(building.is_bitcode('hello_bitcode.o'))
8801+
self.assertTrue(is_bitcode('hello_bitcode.o'))
87848802

87858803
print('use bitcode object (LTO)')
87868804
self.run_process([EMXX, 'hello_bitcode.o'] + args + ['-flto'])
@@ -8813,7 +8831,7 @@ def test_lto_flags(self):
88138831
(['-sWASM_OBJECT_FILES'], False),
88148832
]:
88158833
self.run_process([EMCC, test_file('hello_world.c')] + flags + ['-c', '-o', 'a.o'])
8816-
seen_bitcode = building.is_bitcode('a.o')
8834+
seen_bitcode = is_bitcode('a.o')
88178835
self.assertEqual(expect_bitcode, seen_bitcode, 'must emit LTO-capable bitcode when flags indicate so (%s)' % str(flags))
88188836

88198837
# We have LTO tests covered in 'wasmltoN' targets in test_core.py, but they
@@ -10181,17 +10199,17 @@ def test_is_bitcode(self):
1018110199

1018210200
with open(fname, 'wb') as f:
1018310201
f.write(b'foo')
10184-
self.assertFalse(building.is_bitcode(fname))
10202+
self.assertFalse(is_bitcode(fname))
1018510203

1018610204
with open(fname, 'wb') as f:
1018710205
f.write(b'\xDE\xC0\x17\x0B')
1018810206
f.write(16 * b'\x00')
1018910207
f.write(b'BC')
10190-
self.assertTrue(building.is_bitcode(fname))
10208+
self.assertTrue(is_bitcode(fname))
1019110209

1019210210
with open(fname, 'wb') as f:
1019310211
f.write(b'BC')
10194-
self.assertTrue(building.is_bitcode(fname))
10212+
self.assertTrue(is_bitcode(fname))
1019510213

1019610214
def test_is_ar(self):
1019710215
fname = 'tmp.a'
@@ -11987,7 +12005,7 @@ def test_bitcode_input(self):
1198712005
# Verify that bitcode files are accepted as input
1198812006
create_file('main.c', 'void foo(); int main() { return 0; }')
1198912007
self.run_process([EMCC, '-emit-llvm', '-c', '-o', 'main.bc', 'main.c'])
11990-
self.assertTrue(building.is_bitcode('main.bc'))
12008+
self.assertTrue(is_bitcode('main.bc'))
1199112009
self.run_process([EMCC, '-c', '-o', 'main.o', 'main.bc'])
1199212010
self.assertTrue(building.is_wasm('main.o'))
1199312011

tools/building.py

Lines changed: 4 additions & 116 deletions
Original file line numberDiff line numberDiff line change
@@ -32,7 +32,7 @@
3232
from .shared import LLVM_DWARFDUMP, demangle_c_symbol_name
3333
from .shared import get_emscripten_temp_dir, exe_suffix, is_c_symbol
3434
from .utils import WINDOWS
35-
from .settings import settings, default_setting
35+
from .settings import settings
3636
from .feature_matrix import UNSUPPORTED
3737

3838
logger = logging.getLogger('building')
@@ -46,20 +46,6 @@
4646
user_requested_exports: Set[str] = set()
4747

4848

49-
# .. but for Popen, we cannot have doublequotes, so provide functionality to
50-
# remove them when needed.
51-
def remove_quotes(arg):
52-
if isinstance(arg, list):
53-
return [remove_quotes(a) for a in arg]
54-
55-
if arg.startswith('"') and arg.endswith('"'):
56-
return arg[1:-1].replace('\\"', '"')
57-
elif arg.startswith("'") and arg.endswith("'"):
58-
return arg[1:-1].replace("\\'", "'")
59-
else:
60-
return arg
61-
62-
6349
def get_building_env():
6450
cache.ensure()
6551
env = os.environ.copy()
@@ -1063,34 +1049,15 @@ def handle_final_wasm_symbols(wasm_file, symbols_file, debug_info):
10631049

10641050

10651051
def is_ar(filename):
1052+
"""Return True if a the given filename is an ar archive, False otherwise.
1053+
"""
10661054
try:
1067-
if _is_ar_cache.get(filename):
1068-
return _is_ar_cache[filename]
10691055
header = open(filename, 'rb').read(8)
1070-
sigcheck = header in (b'!<arch>\n', b'!<thin>\n')
1071-
_is_ar_cache[filename] = sigcheck
1072-
return sigcheck
10731056
except Exception as e:
10741057
logger.debug('is_ar failed to test whether file \'%s\' is a llvm archive file! Failed on exception: %s' % (filename, e))
10751058
return False
10761059

1077-
1078-
def is_bitcode(filename):
1079-
try:
1080-
# look for magic signature
1081-
b = open(filename, 'rb').read(4)
1082-
if b[:2] == b'BC':
1083-
return True
1084-
# on macOS, there is a 20-byte prefix which starts with little endian
1085-
# encoding of 0x0B17C0DE
1086-
elif b == b'\xDE\xC0\x17\x0B':
1087-
b = bytearray(open(filename, 'rb').read(22))
1088-
return b[20:] == b'BC'
1089-
except IndexError:
1090-
# not enough characters in the input
1091-
# note that logging will be done on the caller function
1092-
pass
1093-
return False
1060+
return header in (b'!<arch>\n', b'!<thin>\n')
10941061

10951062

10961063
def is_wasm(filename):
@@ -1113,85 +1080,6 @@ def is_wasm_dylib(filename):
11131080
return False
11141081

11151082

1116-
def map_to_js_libs(library_name):
1117-
"""Given the name of a special Emscripten-implemented system library, returns an
1118-
pair containing
1119-
1. Array of absolute paths to JS library files, inside emscripten/src/ that corresponds to the
1120-
library name. `None` means there is no mapping and the library will be processed by the linker
1121-
as a require for normal native library.
1122-
2. Optional name of a corresponding native library to link in.
1123-
"""
1124-
# Some native libraries are implemented in Emscripten as system side JS libraries
1125-
library_map = {
1126-
'embind': ['embind/embind.js', 'embind/emval.js'],
1127-
'EGL': ['library_egl.js'],
1128-
'GL': ['library_webgl.js', 'library_html5_webgl.js'],
1129-
'webgl.js': ['library_webgl.js', 'library_html5_webgl.js'],
1130-
'GLESv2': ['library_webgl.js'],
1131-
# N.b. there is no GLESv3 to link to (note [f] in https://www.khronos.org/registry/implementers_guide.html)
1132-
'GLEW': ['library_glew.js'],
1133-
'glfw': ['library_glfw.js'],
1134-
'glfw3': ['library_glfw.js'],
1135-
'GLU': [],
1136-
'glut': ['library_glut.js'],
1137-
'openal': ['library_openal.js'],
1138-
'X11': ['library_xlib.js'],
1139-
'SDL': ['library_sdl.js'],
1140-
'uuid': ['library_uuid.js'],
1141-
'fetch': ['library_fetch.js'],
1142-
'websocket': ['library_websocket.js'],
1143-
# These 4 libraries are separate under glibc but are all rolled into
1144-
# libc with musl. For compatibility with glibc we just ignore them
1145-
# completely.
1146-
'dl': [],
1147-
'm': [],
1148-
'rt': [],
1149-
'pthread': [],
1150-
# This is the name of GNU's C++ standard library. We ignore it here
1151-
# for compatibility with GNU toolchains.
1152-
'stdc++': [],
1153-
}
1154-
settings_map = {
1155-
'glfw': {'USE_GLFW': 2},
1156-
'glfw3': {'USE_GLFW': 3},
1157-
'SDL': {'USE_SDL': 1},
1158-
}
1159-
1160-
if library_name in settings_map:
1161-
for key, value in settings_map[library_name].items():
1162-
default_setting(key, value)
1163-
1164-
if library_name in library_map:
1165-
libs = library_map[library_name]
1166-
logger.debug('Mapping library `%s` to JS libraries: %s' % (library_name, libs))
1167-
return libs
1168-
1169-
if library_name.endswith('.js') and os.path.isfile(path_from_root('src', f'library_{library_name}')):
1170-
return [f'library_{library_name}']
1171-
1172-
return None
1173-
1174-
1175-
# Map a linker flag to a settings. This lets a user write -lSDL2 and it will
1176-
# have the same effect as -sUSE_SDL=2.
1177-
def map_and_apply_to_settings(library_name):
1178-
# most libraries just work, because the -l name matches the name of the
1179-
# library we build. however, if a library has variations, which cause us to
1180-
# build multiple versions with multiple names, then we need this mechanism.
1181-
library_map = {
1182-
# SDL2_mixer's built library name contains the specific codecs built in.
1183-
'SDL2_mixer': [('USE_SDL_MIXER', 2)],
1184-
}
1185-
1186-
if library_name in library_map:
1187-
for key, value in library_map[library_name]:
1188-
logger.debug('Mapping library `%s` to settings changes: %s = %s' % (library_name, key, value))
1189-
setattr(settings, key, value)
1190-
return True
1191-
1192-
return False
1193-
1194-
11951083
def emit_wasm_source_map(wasm_file, map_file, final_wasm):
11961084
# source file paths must be relative to the location of the map (which is
11971085
# emitted alongside the wasm)

tools/link.py

Lines changed: 96 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1978,6 +1978,20 @@ def phase_embind_aot(wasm_target, js_syms):
19781978
write_file(final_js, src)
19791979

19801980

1981+
# for Popen, we cannot have doublequotes, so provide functionality to
1982+
# remove them when needed.
1983+
def remove_quotes(arg):
1984+
if isinstance(arg, list):
1985+
return [remove_quotes(a) for a in arg]
1986+
1987+
if arg.startswith('"') and arg.endswith('"'):
1988+
return arg[1:-1].replace('\\"', '"')
1989+
elif arg.startswith("'") and arg.endswith("'"):
1990+
return arg[1:-1].replace("\\'", "'")
1991+
else:
1992+
return arg
1993+
1994+
19811995
@ToolchainProfiler.profile_block('source transforms')
19821996
def phase_source_transforms(options):
19831997
# Apply a source code transformation, if requested
@@ -1986,7 +2000,7 @@ def phase_source_transforms(options):
19862000
final_js += '.tr.js'
19872001
posix = not shared.WINDOWS
19882002
logger.debug('applying transform: %s', options.js_transform)
1989-
shared.check_call(building.remove_quotes(shlex.split(options.js_transform, posix=posix) + [os.path.abspath(final_js)]))
2003+
shared.check_call(remove_quotes(shlex.split(options.js_transform, posix=posix) + [os.path.abspath(final_js)]))
19902004
save_intermediate('transformed')
19912005

19922006

@@ -2637,6 +2651,85 @@ def find_library(lib, lib_dirs):
26372651
return None
26382652

26392653

2654+
def map_to_js_libs(library_name):
2655+
"""Given the name of a special Emscripten-implemented system library, returns an
2656+
pair containing
2657+
1. Array of absolute paths to JS library files, inside emscripten/src/ that corresponds to the
2658+
library name. `None` means there is no mapping and the library will be processed by the linker
2659+
as a require for normal native library.
2660+
2. Optional name of a corresponding native library to link in.
2661+
"""
2662+
# Some native libraries are implemented in Emscripten as system side JS libraries
2663+
library_map = {
2664+
'embind': ['embind/embind.js', 'embind/emval.js'],
2665+
'EGL': ['library_egl.js'],
2666+
'GL': ['library_webgl.js', 'library_html5_webgl.js'],
2667+
'webgl.js': ['library_webgl.js', 'library_html5_webgl.js'],
2668+
'GLESv2': ['library_webgl.js'],
2669+
# N.b. there is no GLESv3 to link to (note [f] in https://www.khronos.org/registry/implementers_guide.html)
2670+
'GLEW': ['library_glew.js'],
2671+
'glfw': ['library_glfw.js'],
2672+
'glfw3': ['library_glfw.js'],
2673+
'GLU': [],
2674+
'glut': ['library_glut.js'],
2675+
'openal': ['library_openal.js'],
2676+
'X11': ['library_xlib.js'],
2677+
'SDL': ['library_sdl.js'],
2678+
'uuid': ['library_uuid.js'],
2679+
'fetch': ['library_fetch.js'],
2680+
'websocket': ['library_websocket.js'],
2681+
# These 4 libraries are separate under glibc but are all rolled into
2682+
# libc with musl. For compatibility with glibc we just ignore them
2683+
# completely.
2684+
'dl': [],
2685+
'm': [],
2686+
'rt': [],
2687+
'pthread': [],
2688+
# This is the name of GNU's C++ standard library. We ignore it here
2689+
# for compatibility with GNU toolchains.
2690+
'stdc++': [],
2691+
}
2692+
settings_map = {
2693+
'glfw': {'USE_GLFW': 2},
2694+
'glfw3': {'USE_GLFW': 3},
2695+
'SDL': {'USE_SDL': 1},
2696+
}
2697+
2698+
if library_name in settings_map:
2699+
for key, value in settings_map[library_name].items():
2700+
default_setting(key, value)
2701+
2702+
if library_name in library_map:
2703+
libs = library_map[library_name]
2704+
logger.debug('Mapping library `%s` to JS libraries: %s' % (library_name, libs))
2705+
return libs
2706+
2707+
if library_name.endswith('.js') and os.path.isfile(utils.path_from_root('src', f'library_{library_name}')):
2708+
return [f'library_{library_name}']
2709+
2710+
return None
2711+
2712+
2713+
# Map a linker flag to a settings. This lets a user write -lSDL2 and it will
2714+
# have the same effect as -sUSE_SDL=2.
2715+
def map_and_apply_to_settings(library_name):
2716+
# most libraries just work, because the -l name matches the name of the
2717+
# library we build. however, if a library has variations, which cause us to
2718+
# build multiple versions with multiple names, then we need this mechanism.
2719+
library_map = {
2720+
# SDL2_mixer's built library name contains the specific codecs built in.
2721+
'SDL2_mixer': [('USE_SDL_MIXER', 2)],
2722+
}
2723+
2724+
if library_name in library_map:
2725+
for key, value in library_map[library_name]:
2726+
logger.debug('Mapping library `%s` to settings changes: %s = %s' % (library_name, key, value))
2727+
setattr(settings, key, value)
2728+
return True
2729+
2730+
return False
2731+
2732+
26402733
def process_libraries(state, linker_inputs):
26412734
new_flags = []
26422735
libraries = []
@@ -2652,7 +2745,7 @@ def process_libraries(state, linker_inputs):
26522745

26532746
logger.debug('looking for library "%s"', lib)
26542747

2655-
js_libs = building.map_to_js_libs(lib)
2748+
js_libs = map_to_js_libs(lib)
26562749
if js_libs is not None:
26572750
libraries += [(i, js_lib) for js_lib in js_libs]
26582751

@@ -2667,7 +2760,7 @@ def process_libraries(state, linker_inputs):
26672760
if js_libs is not None:
26682761
continue
26692762

2670-
if building.map_and_apply_to_settings(lib):
2763+
if map_and_apply_to_settings(lib):
26712764
continue
26722765

26732766
path = None

0 commit comments

Comments
 (0)