@@ -75,11 +75,7 @@ const getZeroDelayTimeout = () =>
75
75
76
76
const mockCalculateContentCRC32Mock = ( ) => {
77
77
mockCalculateContentCRC32 . mockReset ( ) ;
78
- mockCalculateContentCRC32 . mockResolvedValue ( {
79
- checksumArrayBuffer : new ArrayBuffer ( 0 ) ,
80
- checksum : 'mockChecksum' ,
81
- seed : 0 ,
82
- } ) ;
78
+ mockCalculateContentCRC32 . mockResolvedValue ( 'mockChecksum' ) ;
83
79
} ;
84
80
const mockCalculateContentCRC32Reset = ( ) => {
85
81
mockCalculateContentCRC32 . mockReset ( ) ;
@@ -276,14 +272,25 @@ describe('getMultipartUploadHandlers with key', () => {
276
272
'file' ,
277
273
new File ( [ getBlob ( 8 * MB ) ] , 'someName' ) ,
278
274
[ 'JCnBsQ==' , 'HELzGQ==' ] ,
275
+ '/YBlgg==' ,
276
+ ] ,
277
+ [ 'blob' , getBlob ( 8 * MB ) , [ 'JCnBsQ==' , 'HELzGQ==' ] , '/YBlgg==' ] ,
278
+ [ 'string' , 'Ü' . repeat ( 4 * MB ) , [ 'DL735w==' , 'Akga7g==' ] , 'dPB9Lw==' ] ,
279
+ [
280
+ 'arrayBuffer' ,
281
+ new ArrayBuffer ( 8 * MB ) ,
282
+ [ 'yTuzdQ==' , 'eXJPxg==' ] ,
283
+ 'GtK8RQ==' ,
284
+ ] ,
285
+ [
286
+ 'arrayBufferView' ,
287
+ new Uint8Array ( 8 * MB ) ,
288
+ [ 'yTuzdQ==' , 'eXJPxg==' ] ,
289
+ 'GtK8RQ==' ,
279
290
] ,
280
- [ 'blob' , getBlob ( 8 * MB ) , [ 'JCnBsQ==' , 'HELzGQ==' ] ] ,
281
- [ 'string' , 'Ü' . repeat ( 4 * MB ) , [ 'DL735w==' , 'Akga7g==' ] ] ,
282
- [ 'arrayBuffer' , new ArrayBuffer ( 8 * MB ) , [ 'yTuzdQ==' , 'eXJPxg==' ] ] ,
283
- [ 'arrayBufferView' , new Uint8Array ( 8 * MB ) , [ 'yTuzdQ==' , 'eXJPxg==' ] ] ,
284
291
] ) (
285
292
`should create crc32 for %s type body` ,
286
- async ( _ , twoPartsPayload , expectedCrc32 ) => {
293
+ async ( _ , twoPartsPayload , expectedCrc32 , finalCrc32 ) => {
287
294
mockMultipartUploadSuccess ( ) ;
288
295
const { multipartUploadJob } = getMultipartUploadHandlers (
289
296
{
@@ -298,17 +305,12 @@ describe('getMultipartUploadHandlers with key', () => {
298
305
await multipartUploadJob ( ) ;
299
306
300
307
/**
301
- * final crc32 calculation calls calculateContentCRC32 3 times
302
- * 1 time for each of the 2 parts
303
- * 1 time to combine the resulting hash for each of the two parts
304
- *
305
- * uploading each part calls calculateContentCRC32 1 time each
306
- *
307
- * 1 time for optionsHash
308
- *
309
- * these steps results in 6 calls in total
308
+ * `calculateContentCRC32` is called 4 times with Full-body checksum:
309
+ * * 1 time when calculating final crc32 for the whole object to be uploaded.
310
+ * * 1 time when calculating optionsHash.
311
+ * * 1 time each when uploading part calls, 2 calls in total.
310
312
*/
311
- expect ( calculateContentCRC32 ) . toHaveBeenCalledTimes ( 6 ) ;
313
+ expect ( calculateContentCRC32 ) . toHaveBeenCalledTimes ( 4 ) ;
312
314
expect ( calculateContentMd5 ) . not . toHaveBeenCalled ( ) ;
313
315
expect ( mockUploadPart ) . toHaveBeenCalledTimes ( 2 ) ;
314
316
expect ( mockUploadPart ) . toHaveBeenCalledWith (
@@ -319,6 +321,13 @@ describe('getMultipartUploadHandlers with key', () => {
319
321
expect . anything ( ) ,
320
322
expect . objectContaining ( { ChecksumCRC32 : expectedCrc32 [ 1 ] } ) ,
321
323
) ;
324
+ expect ( mockCompleteMultipartUpload ) . toHaveBeenCalledWith (
325
+ expect . anything ( ) ,
326
+ expect . objectContaining ( {
327
+ ChecksumCRC32 : finalCrc32 ,
328
+ ChecksumType : 'FULL_OBJECT' ,
329
+ } ) ,
330
+ ) ;
322
331
} ,
323
332
) ;
324
333
@@ -389,7 +398,7 @@ describe('getMultipartUploadHandlers with key', () => {
389
398
file . size ,
390
399
) ;
391
400
await multipartUploadJob ( ) ;
392
- expect ( file . slice ) . toHaveBeenCalledTimes ( 10_000 * 2 ) ; // S3 limit of parts count double for crc32 calculations
401
+ expect ( file . slice ) . toHaveBeenCalledTimes ( 10_000 ) ;
393
402
expect ( mockCreateMultipartUpload ) . toHaveBeenCalledTimes ( 1 ) ;
394
403
expect ( mockUploadPart ) . toHaveBeenCalledTimes ( 10_000 ) ;
395
404
expect ( mockCompleteMultipartUpload ) . toHaveBeenCalledTimes ( 1 ) ;
@@ -565,6 +574,7 @@ describe('getMultipartUploadHandlers with key', () => {
565
574
bucket,
566
575
key : defaultKey ,
567
576
finalCrc32 : 'mock-crc32' ,
577
+ lastTouched : Date . now ( ) - 2 * 60 * 1000 , // 2 mins ago
568
578
} ,
569
579
} ) ,
570
580
) ;
@@ -651,7 +661,7 @@ describe('getMultipartUploadHandlers with key', () => {
651
661
await multipartUploadJob ( ) ;
652
662
expect ( mockCalculateContentCRC32 ) . toHaveBeenNthCalledWith (
653
663
1 ,
654
- JSON . stringify ( serializableOptions ) ,
664
+ JSON . stringify ( { ... serializableOptions , checksumType : 'FULL_OBJECT' } ) ,
655
665
) ;
656
666
} ) ;
657
667
@@ -734,14 +744,46 @@ describe('getMultipartUploadHandlers with key', () => {
734
744
] ) ;
735
745
} ) ;
736
746
747
+ it ( 'should clean any outdated in-progress uploads' , async ( ) => {
748
+ mockDefaultStorage . getItem . mockResolvedValue (
749
+ JSON . stringify ( {
750
+ 'other-outdated-update' : {
751
+ uploadId : '000' ,
752
+ bucket,
753
+ key : defaultKey ,
754
+ lastTouched : Date . now ( ) - 2 * 60 * 60 * 1000 , // 2 hours ago
755
+ } ,
756
+ } ) ,
757
+ ) ;
758
+ mockMultipartUploadSuccess ( ) ;
759
+ mockListParts . mockResolvedValueOnce ( { Parts : [ ] , $metadata : { } } ) ;
760
+ const size = 8 * MB ;
761
+ const { multipartUploadJob } = getMultipartUploadHandlers (
762
+ {
763
+ key : defaultKey ,
764
+ data : new File ( [ new ArrayBuffer ( size ) ] , 'someName' ) ,
765
+ options : {
766
+ resumableUploadsCache : mockDefaultStorage ,
767
+ } ,
768
+ } ,
769
+ size ,
770
+ ) ;
771
+ await multipartUploadJob ( ) ;
772
+ // 1 for evicting outdated cached uploads;
773
+ // 1 for caching upload task;
774
+ // 1 for remove cache after upload is completed
775
+ expect ( mockDefaultStorage . setItem ) . toHaveBeenCalledTimes ( 3 ) ;
776
+ expect ( mockDefaultStorage . setItem . mock . calls [ 0 ] [ 1 ] ) . toEqual ( '{}' ) ;
777
+ } ) ;
778
+
737
779
it ( 'should send listParts request if the upload task is cached' , async ( ) => {
738
780
mockDefaultStorage . getItem . mockResolvedValue (
739
781
JSON . stringify ( {
740
782
[ defaultCacheKey ] : {
741
783
uploadId : 'uploadId' ,
742
784
bucket,
743
785
key : defaultKey ,
744
- lastModified : Date . now ( ) ,
786
+ lastTouched : Date . now ( ) ,
745
787
} ,
746
788
} ) ,
747
789
) ;
@@ -960,6 +1002,7 @@ describe('getMultipartUploadHandlers with key', () => {
960
1002
uploadId : 'uploadId' ,
961
1003
bucket,
962
1004
key : defaultKey ,
1005
+ lastTouched : Date . now ( ) ,
963
1006
} ,
964
1007
} ) ,
965
1008
) ;
@@ -1090,14 +1133,25 @@ describe('getMultipartUploadHandlers with path', () => {
1090
1133
'file' ,
1091
1134
new File ( [ getBlob ( 8 * MB ) ] , 'someName' ) ,
1092
1135
[ 'JCnBsQ==' , 'HELzGQ==' ] ,
1136
+ '/YBlgg==' ,
1137
+ ] ,
1138
+ [ 'blob' , getBlob ( 8 * MB ) , [ 'JCnBsQ==' , 'HELzGQ==' ] , '/YBlgg==' ] ,
1139
+ [ 'string' , 'Ü' . repeat ( 4 * MB ) , [ 'DL735w==' , 'Akga7g==' ] , 'dPB9Lw==' ] ,
1140
+ [
1141
+ 'arrayBuffer' ,
1142
+ new ArrayBuffer ( 8 * MB ) ,
1143
+ [ 'yTuzdQ==' , 'eXJPxg==' ] ,
1144
+ 'GtK8RQ==' ,
1145
+ ] ,
1146
+ [
1147
+ 'arrayBufferView' ,
1148
+ new Uint8Array ( 8 * MB ) ,
1149
+ [ 'yTuzdQ==' , 'eXJPxg==' ] ,
1150
+ 'GtK8RQ==' ,
1093
1151
] ,
1094
- [ 'blob' , getBlob ( 8 * MB ) , [ 'JCnBsQ==' , 'HELzGQ==' ] ] ,
1095
- [ 'string' , 'Ü' . repeat ( 4 * MB ) , [ 'DL735w==' , 'Akga7g==' ] ] ,
1096
- [ 'arrayBuffer' , new ArrayBuffer ( 8 * MB ) , [ 'yTuzdQ==' , 'eXJPxg==' ] ] ,
1097
- [ 'arrayBufferView' , new Uint8Array ( 8 * MB ) , [ 'yTuzdQ==' , 'eXJPxg==' ] ] ,
1098
1152
] ) (
1099
1153
`should create crc32 for %s type body` ,
1100
- async ( _ , twoPartsPayload , expectedCrc32 ) => {
1154
+ async ( _ , twoPartsPayload , expectedCrc32 , finalCrc32 ) => {
1101
1155
mockMultipartUploadSuccess ( ) ;
1102
1156
const { multipartUploadJob } = getMultipartUploadHandlers (
1103
1157
{
@@ -1112,17 +1166,12 @@ describe('getMultipartUploadHandlers with path', () => {
1112
1166
await multipartUploadJob ( ) ;
1113
1167
1114
1168
/**
1115
- * final crc32 calculation calls calculateContentCRC32 3 times
1116
- * 1 time for each of the 2 parts
1117
- * 1 time to combine the resulting hash for each of the two parts
1118
- *
1119
- * uploading each part calls calculateContentCRC32 1 time each
1120
- *
1121
- * 1 time for optionsHash
1122
- *
1123
- * these steps results in 6 calls in total
1169
+ * `calculateContentCRC32` is called 4 times with Full-body checksum:
1170
+ * * 1 time when calculating final crc32 for the whole object to be uploaded.
1171
+ * * 1 time when calculating optionsHash.
1172
+ * * 1 time each when uploading part calls, 2 calls in total.
1124
1173
*/
1125
- expect ( calculateContentCRC32 ) . toHaveBeenCalledTimes ( 6 ) ;
1174
+ expect ( calculateContentCRC32 ) . toHaveBeenCalledTimes ( 4 ) ;
1126
1175
expect ( calculateContentMd5 ) . not . toHaveBeenCalled ( ) ;
1127
1176
expect ( mockUploadPart ) . toHaveBeenCalledTimes ( 2 ) ;
1128
1177
expect ( mockUploadPart ) . toHaveBeenCalledWith (
@@ -1133,6 +1182,13 @@ describe('getMultipartUploadHandlers with path', () => {
1133
1182
expect . anything ( ) ,
1134
1183
expect . objectContaining ( { ChecksumCRC32 : expectedCrc32 [ 1 ] } ) ,
1135
1184
) ;
1185
+ expect ( mockCompleteMultipartUpload ) . toHaveBeenCalledWith (
1186
+ expect . anything ( ) ,
1187
+ expect . objectContaining ( {
1188
+ ChecksumCRC32 : finalCrc32 ,
1189
+ ChecksumType : 'FULL_OBJECT' ,
1190
+ } ) ,
1191
+ ) ;
1136
1192
} ,
1137
1193
) ;
1138
1194
@@ -1203,7 +1259,7 @@ describe('getMultipartUploadHandlers with path', () => {
1203
1259
file . size ,
1204
1260
) ;
1205
1261
await multipartUploadJob ( ) ;
1206
- expect ( file . slice ) . toHaveBeenCalledTimes ( 10_000 * 2 ) ; // S3 limit of parts count double for crc32 calculations
1262
+ expect ( file . slice ) . toHaveBeenCalledTimes ( 10_000 ) ;
1207
1263
expect ( mockCreateMultipartUpload ) . toHaveBeenCalledTimes ( 1 ) ;
1208
1264
expect ( mockUploadPart ) . toHaveBeenCalledTimes ( 10_000 ) ;
1209
1265
expect ( mockCompleteMultipartUpload ) . toHaveBeenCalledTimes ( 1 ) ;
@@ -1413,6 +1469,7 @@ describe('getMultipartUploadHandlers with path', () => {
1413
1469
bucket,
1414
1470
key : defaultKey ,
1415
1471
finalCrc32 : 'mock-crc32' ,
1472
+ lastTouched : Date . now ( ) ,
1416
1473
} ,
1417
1474
} ) ,
1418
1475
) ;
@@ -1519,7 +1576,7 @@ describe('getMultipartUploadHandlers with path', () => {
1519
1576
await multipartUploadJob ( ) ;
1520
1577
expect ( mockCalculateContentCRC32 ) . toHaveBeenNthCalledWith (
1521
1578
1 ,
1522
- JSON . stringify ( serializableOptions ) ,
1579
+ JSON . stringify ( { ... serializableOptions , checksumType : 'FULL_OBJECT' } ) ,
1523
1580
) ;
1524
1581
} ) ;
1525
1582
@@ -1583,14 +1640,46 @@ describe('getMultipartUploadHandlers with path', () => {
1583
1640
] ) ;
1584
1641
} ) ;
1585
1642
1643
+ it ( 'should clean any outdated in-progress uploads' , async ( ) => {
1644
+ mockDefaultStorage . getItem . mockResolvedValue (
1645
+ JSON . stringify ( {
1646
+ 'other-outdated-update' : {
1647
+ uploadId : '000' ,
1648
+ bucket,
1649
+ key : defaultKey ,
1650
+ lastTouched : Date . now ( ) - 2 * 60 * 60 * 1000 , // 2 hours ago
1651
+ } ,
1652
+ } ) ,
1653
+ ) ;
1654
+ mockMultipartUploadSuccess ( ) ;
1655
+ mockListParts . mockResolvedValueOnce ( { Parts : [ ] , $metadata : { } } ) ;
1656
+ const size = 8 * MB ;
1657
+ const { multipartUploadJob } = getMultipartUploadHandlers (
1658
+ {
1659
+ path : testPath ,
1660
+ data : new File ( [ new ArrayBuffer ( size ) ] , 'someName' ) ,
1661
+ options : {
1662
+ resumableUploadsCache : mockDefaultStorage ,
1663
+ } ,
1664
+ } ,
1665
+ size ,
1666
+ ) ;
1667
+ await multipartUploadJob ( ) ;
1668
+ // 1 for evicting outdated cached uploads;
1669
+ // 1 for caching upload task;
1670
+ // 1 for remove cache after upload is completed
1671
+ expect ( mockDefaultStorage . setItem ) . toHaveBeenCalledTimes ( 3 ) ;
1672
+ expect ( mockDefaultStorage . setItem . mock . calls [ 0 ] [ 1 ] ) . toEqual ( '{}' ) ;
1673
+ } ) ;
1674
+
1586
1675
it ( 'should send listParts request if the upload task is cached' , async ( ) => {
1587
1676
mockDefaultStorage . getItem . mockResolvedValue (
1588
1677
JSON . stringify ( {
1589
1678
[ testPathCacheKey ] : {
1590
1679
uploadId : 'uploadId' ,
1591
1680
bucket,
1592
1681
key : testPath ,
1593
- lastModified : Date . now ( ) ,
1682
+ lastTouched : Date . now ( ) ,
1594
1683
} ,
1595
1684
} ) ,
1596
1685
) ;
@@ -1808,6 +1897,7 @@ describe('getMultipartUploadHandlers with path', () => {
1808
1897
uploadId : 'uploadId' ,
1809
1898
bucket,
1810
1899
key : testPath ,
1900
+ lastTouched : Date . now ( ) ,
1811
1901
} ,
1812
1902
} ) ,
1813
1903
) ;
0 commit comments