Skip to content

Commit 5f2125f

Browse files
mviethmvieth
authored
Enable writing very large PCD files on Windows (#5675)
* Enable writing very large PCD files on Windows CreateFileMapping accepts the file size in two 32 bit, unsigned integer parameters (a high and a low part). Previously, PCL has always set the high part to 0, meaning that the maximal file size was 2^32. To be sure that the shift operator shifts in zeros, the integer has to be unsigned, so data_idx is changed to an unsigned integer everywhere. * Remove return after throw
1 parent ee626d8 commit 5f2125f

File tree

3 files changed

+14
-33
lines changed

3 files changed

+14
-33
lines changed

‎io/include/pcl/io/impl/pcd_io.hpp

Lines changed: 10 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -115,25 +115,22 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
115115
{
116116
PCL_WARN ("[pcl::PCDWriter::writeBinary] Input point cloud has no data!\n");
117117
}
118-
int data_idx = 0;
119118
std::ostringstream oss;
120119
oss << generateHeader<PointT> (cloud) << "DATA binary\n";
121120
oss.flush ();
122-
data_idx = static_cast<int> (oss.tellp ());
121+
const auto data_idx = static_cast<unsigned int> (oss.tellp ());
123122

124123
#ifdef _WIN32
125124
HANDLE h_native_file = CreateFileA (file_name.c_str (), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
126125
if (h_native_file == INVALID_HANDLE_VALUE)
127126
{
128127
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during CreateFile!");
129-
return (-1);
130128
}
131129
#else
132130
int fd = io::raw_open (file_name.c_str (), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
133131
if (fd < 0)
134132
{
135133
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during open!");
136-
return (-1);
137134
}
138135
#endif
139136
// Mandatory lock file
@@ -162,13 +159,17 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
162159

163160
// Prepare the map
164161
#ifdef _WIN32
165-
HANDLE fm = CreateFileMappingA (h_native_file, NULL, PAGE_READWRITE, 0, (DWORD) (data_idx + data_size), NULL);
162+
HANDLE fm = CreateFileMappingA (h_native_file, NULL, PAGE_READWRITE, (DWORD) ((data_idx + data_size) >> 32), (DWORD) (data_idx + data_size), NULL);
166163
if (fm == NULL)
167164
{
168165
throw pcl::IOException("[pcl::PCDWriter::writeBinary] Error during memory map creation ()!");
169-
return (-1);
170166
}
171167
char *map = static_cast<char*>(MapViewOfFile (fm, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, data_idx + data_size));
168+
if (map == NULL)
169+
{
170+
CloseHandle (fm);
171+
throw pcl::IOException("[pcl::PCDWriter::writeBinary] Error mapping view of file!");
172+
}
172173
CloseHandle (fm);
173174

174175
#else
@@ -182,7 +183,6 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
182183
data_idx + data_size, allocate_res, errno, strerror (errno));
183184

184185
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during raw_fallocate ()!");
185-
return (-1);
186186
}
187187

188188
char *map = static_cast<char*> (::mmap (nullptr, data_idx + data_size, PROT_WRITE, MAP_SHARED, fd, 0));
@@ -191,7 +191,6 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
191191
io::raw_close (fd);
192192
resetLockingPermissions (file_name, file_lock);
193193
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during mmap ()!");
194-
return (-1);
195194
}
196195
#endif
197196

@@ -225,7 +224,6 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
225224
io::raw_close (fd);
226225
resetLockingPermissions (file_name, file_lock);
227226
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during munmap ()!");
228-
return (-1);
229227
}
230228
#endif
231229
// Close file
@@ -247,25 +245,22 @@ pcl::PCDWriter::writeBinaryCompressed (const std::string &file_name,
247245
{
248246
PCL_WARN ("[pcl::PCDWriter::writeBinaryCompressed] Input point cloud has no data!\n");
249247
}
250-
int data_idx = 0;
251248
std::ostringstream oss;
252249
oss << generateHeader<PointT> (cloud) << "DATA binary_compressed\n";
253250
oss.flush ();
254-
data_idx = static_cast<int> (oss.tellp ());
251+
const auto data_idx = static_cast<unsigned int> (oss.tellp ());
255252

256253
#ifdef _WIN32
257254
HANDLE h_native_file = CreateFileA (file_name.c_str (), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
258255
if (h_native_file == INVALID_HANDLE_VALUE)
259256
{
260257
throw pcl::IOException ("[pcl::PCDWriter::writeBinaryCompressed] Error during CreateFile!");
261-
return (-1);
262258
}
263259
#else
264260
int fd = io::raw_open (file_name.c_str (), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
265261
if (fd < 0)
266262
{
267263
throw pcl::IOException ("[pcl::PCDWriter::writeBinaryCompressed] Error during open!");
268-
return (-1);
269264
}
270265
#endif
271266

@@ -392,7 +387,6 @@ pcl::PCDWriter::writeBinaryCompressed (const std::string &file_name,
392387
compressed_final_size, allocate_res, errno, strerror (errno));
393388

394389
throw pcl::IOException ("[pcl::PCDWriter::writeBinaryCompressed] Error during raw_fallocate ()!");
395-
return (-1);
396390
}
397391

398392
char *map = static_cast<char*> (::mmap (nullptr, compressed_final_size, PROT_WRITE, MAP_SHARED, fd, 0));
@@ -401,7 +395,6 @@ pcl::PCDWriter::writeBinaryCompressed (const std::string &file_name,
401395
io::raw_close (fd);
402396
resetLockingPermissions (file_name, file_lock);
403397
throw pcl::IOException ("[pcl::PCDWriter::writeBinaryCompressed] Error during mmap ()!");
404-
return (-1);
405398
}
406399
#endif
407400

@@ -425,7 +418,6 @@ pcl::PCDWriter::writeBinaryCompressed (const std::string &file_name,
425418
io::raw_close (fd);
426419
resetLockingPermissions (file_name, file_lock);
427420
throw pcl::IOException ("[pcl::PCDWriter::writeBinaryCompressed] Error during munmap ()!");
428-
return (-1);
429421
}
430422
#endif
431423

@@ -455,7 +447,6 @@ pcl::PCDWriter::writeASCII (const std::string &file_name, const pcl::PointCloud<
455447
if (cloud.width * cloud.height != cloud.size ())
456448
{
457449
throw pcl::IOException ("[pcl::PCDWriter::writeASCII] Number of points different than width * height!");
458-
return (-1);
459450
}
460451

461452
std::ofstream fs;
@@ -464,7 +455,6 @@ pcl::PCDWriter::writeASCII (const std::string &file_name, const pcl::PointCloud<
464455
if (!fs.is_open () || fs.fail ())
465456
{
466457
throw pcl::IOException ("[pcl::PCDWriter::writeASCII] Could not open file for writing!");
467-
return (-1);
468458
}
469459

470460
// Mandatory lock file
@@ -625,25 +615,22 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
625615
{
626616
PCL_WARN ("[pcl::PCDWriter::writeBinary] Input point cloud has no data or empty indices given!\n");
627617
}
628-
int data_idx = 0;
629618
std::ostringstream oss;
630619
oss << generateHeader<PointT> (cloud, static_cast<int> (indices.size ())) << "DATA binary\n";
631620
oss.flush ();
632-
data_idx = static_cast<int> (oss.tellp ());
621+
const auto data_idx = static_cast<unsigned int> (oss.tellp ());
633622

634623
#ifdef _WIN32
635624
HANDLE h_native_file = CreateFileA (file_name.c_str (), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
636625
if (h_native_file == INVALID_HANDLE_VALUE)
637626
{
638627
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during CreateFile!");
639-
return (-1);
640628
}
641629
#else
642630
int fd = io::raw_open (file_name.c_str (), O_RDWR | O_CREAT | O_TRUNC, S_IRUSR | S_IWUSR | S_IRGRP | S_IROTH);
643631
if (fd < 0)
644632
{
645633
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during open!");
646-
return (-1);
647634
}
648635
#endif
649636
// Mandatory lock file
@@ -672,7 +659,7 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
672659

673660
// Prepare the map
674661
#ifdef _WIN32
675-
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, 0, data_idx + data_size, NULL);
662+
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, (DWORD) ((data_idx + data_size) >> 32), (DWORD) (data_idx + data_size), NULL);
676663
char *map = static_cast<char*>(MapViewOfFile (fm, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, data_idx + data_size));
677664
CloseHandle (fm);
678665

@@ -687,7 +674,6 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
687674
data_idx + data_size, allocate_res, errno, strerror (errno));
688675

689676
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during raw_fallocate ()!");
690-
return (-1);
691677
}
692678

693679
char *map = static_cast<char*> (::mmap (nullptr, data_idx + data_size, PROT_WRITE, MAP_SHARED, fd, 0));
@@ -696,7 +682,6 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
696682
io::raw_close (fd);
697683
resetLockingPermissions (file_name, file_lock);
698684
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during mmap ()!");
699-
return (-1);
700685
}
701686
#endif
702687

@@ -730,7 +715,6 @@ pcl::PCDWriter::writeBinary (const std::string &file_name,
730715
io::raw_close (fd);
731716
resetLockingPermissions (file_name, file_lock);
732717
throw pcl::IOException ("[pcl::PCDWriter::writeBinary] Error during munmap ()!");
733-
return (-1);
734718
}
735719
#endif
736720
// Close file
@@ -759,15 +743,13 @@ pcl::PCDWriter::writeASCII (const std::string &file_name,
759743
if (cloud.width * cloud.height != cloud.size ())
760744
{
761745
throw pcl::IOException ("[pcl::PCDWriter::writeASCII] Number of points different than width * height!");
762-
return (-1);
763746
}
764747

765748
std::ofstream fs;
766749
fs.open (file_name.c_str (), std::ios::binary); // Open file
767750
if (!fs.is_open () || fs.fail ())
768751
{
769752
throw pcl::IOException ("[pcl::PCDWriter::writeASCII] Could not open file for writing!");
770-
return (-1);
771753
}
772754

773755
// Mandatory lock file

‎io/src/lzf_image_io.cpp

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -67,7 +67,7 @@ pcl::io::LZFImageWriter::saveImageBlob (const char* data,
6767
HANDLE h_native_file = CreateFile (filename.c_str (), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
6868
if (h_native_file == INVALID_HANDLE_VALUE)
6969
return (false);
70-
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, 0, data_size, NULL);
70+
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, (DWORD) (data_size >> 32), (DWORD) (data_size), NULL);
7171
char *map = static_cast<char*> (MapViewOfFile (fm, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, data_size));
7272
CloseHandle (fm);
7373
std::copy(data, data + data_size, map);

‎io/src/pcd_io.cpp

Lines changed: 3 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -1217,13 +1217,12 @@ pcl::PCDWriter::writeBinary (const std::string &file_name, const pcl::PCLPointCl
12171217
return (-1);
12181218
}
12191219

1220-
std::streamoff data_idx = 0;
12211220
std::ostringstream oss;
12221221
oss.imbue (std::locale::classic ());
12231222

12241223
oss << generateHeaderBinary (cloud, origin, orientation) << "DATA binary\n";
12251224
oss.flush();
1226-
data_idx = static_cast<unsigned int> (oss.tellp ());
1225+
const auto data_idx = static_cast<unsigned int> (oss.tellp ());
12271226

12281227
#ifdef _WIN32
12291228
HANDLE h_native_file = CreateFile (file_name.c_str (), GENERIC_READ | GENERIC_WRITE, 0, NULL, CREATE_ALWAYS, FILE_ATTRIBUTE_NORMAL, NULL);
@@ -1269,7 +1268,7 @@ pcl::PCDWriter::writeBinary (const std::string &file_name, const pcl::PCLPointCl
12691268
#endif
12701269
// Prepare the map
12711270
#ifdef _WIN32
1272-
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, 0, (DWORD) (data_idx + cloud.data.size ()), NULL);
1271+
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, (DWORD) ((data_idx + cloud.data.size ()) >> 32), (DWORD) (data_idx + cloud.data.size ()), NULL);
12731272
char *map = static_cast<char*>(MapViewOfFile (fm, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, data_idx + cloud.data.size ()));
12741273
CloseHandle (fm);
12751274

@@ -1494,7 +1493,7 @@ pcl::PCDWriter::writeBinaryCompressed (const std::string &file_name, const pcl::
14941493

14951494
// Prepare the map
14961495
#ifdef _WIN32
1497-
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, 0, ostr.size (), NULL);
1496+
HANDLE fm = CreateFileMapping (h_native_file, NULL, PAGE_READWRITE, (DWORD) ((ostr.size ()) >> 32), (DWORD) (ostr.size ()), NULL);
14981497
char *map = static_cast<char*> (MapViewOfFile (fm, FILE_MAP_READ | FILE_MAP_WRITE, 0, 0, ostr.size ()));
14991498
CloseHandle (fm);
15001499

0 commit comments

Comments
 (0)