@@ -283,6 +283,71 @@ static void saveThinArchiveToRepro(ArchiveFile const *file) {
283
283
" : Archive::children failed: " + toString (std::move (e)));
284
284
}
285
285
286
+ typedef struct {
287
+ StringRef path;
288
+ LoadType loadType;
289
+ bool isLazy;
290
+ std::optional<MemoryBufferRef> buffer;
291
+ const char *start;
292
+ size_t size;
293
+ } DeferredFile;
294
+
295
+ // Most input files have been mapped but not yet paged in.
296
+ // This code forces the page-ins on multiple threads so
297
+ // the process is not stalled waiting on disk buffer i/o.
298
+ static void multiThreadedPageIn (std::vector<DeferredFile> &deferred) {
299
+ #ifndef _WIN32
300
+ #define MaxReadThreads 200
301
+ typedef struct {
302
+ std::vector<DeferredFile> &deferred;
303
+ size_t counter, total, pageSize;
304
+ pthread_mutex_t mutex;
305
+ } PageInState;
306
+ PageInState state = {deferred, 0 , 0 ,
307
+ llvm::sys::Process::getPageSizeEstimate (),
308
+ pthread_mutex_t ()};
309
+ static size_t totalBytes;
310
+
311
+ pthread_t running[MaxReadThreads];
312
+ if (config->readThreads > MaxReadThreads)
313
+ config->readThreads = MaxReadThreads;
314
+ pthread_mutex_init (&state.mutex , NULL );
315
+
316
+ for (int t = 0 ; t < config->readThreads ; t++)
317
+ pthread_create (
318
+ &running[t], nullptr ,
319
+ [](void *ptr) -> void * {
320
+ PageInState &state = *(PageInState *)ptr;
321
+ while (true ) {
322
+ pthread_mutex_lock (&state.mutex );
323
+ if (state.counter >= state.deferred .size ()) {
324
+ pthread_mutex_unlock (&state.mutex );
325
+ return nullptr ;
326
+ }
327
+ DeferredFile &file = state.deferred [state.counter ];
328
+ state.counter += 1 ;
329
+ pthread_mutex_unlock (&state.mutex );
330
+
331
+ const char *page = file.start , *end = page + file.size ;
332
+ totalBytes += end - page;
333
+
334
+ int t = 0 ; // Reference each page to load it into memory.
335
+ for (; page < end; page += state.pageSize )
336
+ t += *page;
337
+ state.total += t; // Avoids the loop being optimised out.
338
+ }
339
+ },
340
+ &state);
341
+
342
+ for (int t = 0 ; t < config->readThreads ; t++)
343
+ pthread_join (running[t], nullptr );
344
+
345
+ pthread_mutex_destroy (&state.mutex );
346
+ if (0 )
347
+ printf (" multiThreadedPageIn %ld/%ld\n " , totalBytes, deferred.size ());
348
+ #endif
349
+ }
350
+
286
351
static InputFile *processFile (std::optional<MemoryBufferRef> buffer,
287
352
StringRef path, LoadType loadType,
288
353
bool isLazy = false , bool isExplicit = true ,
@@ -367,6 +432,7 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
367
432
// we already found that it contains an ObjC symbol.
368
433
if (readFile (path)) {
369
434
Error e = Error::success ();
435
+ std::vector<DeferredFile> deferredFiles;
370
436
for (const object::Archive::Child &c : file->getArchive ().children (e)) {
371
437
Expected<MemoryBufferRef> mb = c.getMemoryBufferRef ();
372
438
if (!mb) {
@@ -380,6 +446,9 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
380
446
continue ;
381
447
}
382
448
449
+ deferredFiles.push_back ({path, LoadType::CommandLine, false ,
450
+ std::nullopt, mb->getBuffer ().data (),
451
+ mb->getBuffer ().size ()});
383
452
if (!hasObjCSection (*mb))
384
453
continue ;
385
454
if (Error e = file->fetch (c, " -ObjC" ))
@@ -389,6 +458,8 @@ static InputFile *processFile(std::optional<MemoryBufferRef> buffer,
389
458
if (e)
390
459
error (toString (file) +
391
460
" : Archive::children failed: " + toString (std::move (e)));
461
+ if (deferredFiles.size () > 1 )
462
+ multiThreadedPageIn (deferredFiles);
392
463
}
393
464
}
394
465
file->addLazySymbols ();
@@ -450,18 +521,13 @@ static InputFile *addFile(StringRef path, LoadType loadType,
450
521
isBundleLoader, isForceHidden);
451
522
}
452
523
453
- typedef struct {
454
- StringRef path;
455
- LoadType loadType;
456
- bool isLazy;
457
- std::optional<MemoryBufferRef> buffer;
458
- } DeferredFile;
459
-
460
524
static void deferFile (StringRef path, LoadType loadType, bool isLazy,
461
525
std::vector<DeferredFile> &deferred) {
462
526
std::optional<MemoryBufferRef> buffer = readFile (path);
463
527
if (config->readThreads )
464
- deferred.push_back ({path, loadType, isLazy, buffer});
528
+ deferred.push_back ({path, loadType, isLazy, buffer,
529
+ buffer->getBuffer ().data (),
530
+ buffer->getBuffer ().size ()});
465
531
else
466
532
processFile (buffer, path, loadType, isLazy);
467
533
}
@@ -1241,60 +1307,6 @@ static void handleSymbolPatterns(InputArgList &args,
1241
1307
parseSymbolPatternsFile (arg, symbolPatterns);
1242
1308
}
1243
1309
1244
- // Most input files have been mapped but not yet paged in.
1245
- // This code forces the page-ins on multiple threads so
1246
- // the process is not stalled waiting on disk buffer i/o.
1247
- void multiThreadedPageIn (std::vector<DeferredFile> &deferred, int nthreads) {
1248
- #ifndef _WIN32
1249
- #define MaxReadThreads 200
1250
- typedef struct {
1251
- std::vector<DeferredFile> &deferred;
1252
- size_t counter, bytes, total, pageSize;
1253
- pthread_mutex_t mutex;
1254
- } PageInState;
1255
- PageInState state = {
1256
- deferred, 0 , 0 , 0 , llvm::sys::Process::getPageSizeEstimate (),
1257
- pthread_mutex_t ()};
1258
- pthread_mutex_init (&state.mutex , NULL );
1259
-
1260
- pthread_t running[MaxReadThreads];
1261
- if (nthreads > MaxReadThreads)
1262
- nthreads = MaxReadThreads;
1263
-
1264
- for (int t = 0 ; t < nthreads; t++)
1265
- pthread_create (
1266
- &running[t], nullptr ,
1267
- [](void *ptr) -> void * {
1268
- PageInState &state = *(PageInState *)ptr;
1269
- while (true ) {
1270
- pthread_mutex_lock (&state.mutex );
1271
- if (state.counter >= state.deferred .size ()) {
1272
- pthread_mutex_unlock (&state.mutex );
1273
- return nullptr ;
1274
- }
1275
- DeferredFile &file = state.deferred [state.counter ];
1276
- state.counter += 1 ;
1277
- pthread_mutex_unlock (&state.mutex );
1278
-
1279
- const char *page = file.buffer ->getBuffer ().data (),
1280
- *end = page + file.buffer ->getBuffer ().size ();
1281
- state.bytes += end - page;
1282
-
1283
- int t = 0 ; // Reference each page to load it into memory.
1284
- for (; page < end; page += state.pageSize )
1285
- t += *page;
1286
- state.total += t; // Avoids the loop being optimised out.
1287
- }
1288
- },
1289
- &state);
1290
-
1291
- for (int t = 0 ; t < nthreads; t++)
1292
- pthread_join (running[t], nullptr );
1293
-
1294
- pthread_mutex_destroy (&state.mutex );
1295
- #endif
1296
- }
1297
-
1298
1310
void createFiles (const InputArgList &args) {
1299
1311
TimeTraceScope timeScope (" Load input files" );
1300
1312
// This loop should be reserved for options whose exact ordering matters.
@@ -1380,7 +1392,7 @@ void createFiles(const InputArgList &args) {
1380
1392
}
1381
1393
1382
1394
if (config->readThreads ) {
1383
- multiThreadedPageIn (deferredFiles, config-> readThreads );
1395
+ multiThreadedPageIn (deferredFiles);
1384
1396
for (auto &file : deferredFiles)
1385
1397
processFile (file.buffer , file.path , file.loadType , file.isLazy );
1386
1398
}
0 commit comments