Skip to content

Commit b2af630

Browse files
martinlsmMartin Lindström
martinlsm
and
Martin Lindström
authored
Arm backend: Add unit tests for per-channel quantization (#12192)
Add explicit tests for per-channel quantization to all applicable operators (Convolutions and Linear ops). This is achieved by parametrizing the unit tests such that they both run per-channel and per-tensor quantization. ### Test plan The unit tests themselves are changed. Thus the code is tested. Co-authored-by: Martin Lindström <Martin.Lindstroem@arm.com>
1 parent 2871fcf commit b2af630

File tree

7 files changed

+321
-149
lines changed

7 files changed

+321
-149
lines changed

‎backends/arm/test/models/test_mobilenet_v2_arm.py

Lines changed: 15 additions & 6 deletions
Original file line numberDiff line numberDiff line change
@@ -32,21 +32,28 @@
3232
input_t = Tuple[torch.Tensor]
3333

3434

35+
quant_test_data = {
36+
"per_channel_quantization=true": True,
37+
"per_channel_quantization=false": False,
38+
}
39+
40+
3541
def test_mv2_tosa_MI():
3642
pipeline = TosaPipelineMI[input_t](
3743
mv2, model_inputs, aten_op=[], exir_op=[], use_to_edge_transform_and_lower=True
3844
)
3945
pipeline.run()
4046

4147

42-
def test_mv2_tosa_BI():
48+
@common.parametrize("per_channel_quantization", quant_test_data)
49+
def test_mv2_tosa_BI(per_channel_quantization):
4350
pipeline = TosaPipelineBI[input_t](
4451
mv2,
4552
model_inputs,
4653
aten_op=[],
4754
exir_op=[],
4855
use_to_edge_transform_and_lower=True,
49-
per_channel_quantization=True,
56+
per_channel_quantization=per_channel_quantization,
5057
atol=0.25,
5158
qtol=1,
5259
)
@@ -55,15 +62,16 @@ def test_mv2_tosa_BI():
5562

5663
@pytest.mark.slow
5764
@common.XfailIfNoCorstone300
58-
def test_mv2_u55_BI():
65+
@common.parametrize("per_channel_quantization", quant_test_data)
66+
def test_mv2_u55_BI(per_channel_quantization):
5967
pipeline = EthosU55PipelineBI[input_t](
6068
mv2,
6169
model_inputs,
6270
aten_ops=[],
6371
exir_ops=[],
6472
run_on_fvp=True,
6573
use_to_edge_transform_and_lower=True,
66-
per_channel_quantization=True,
74+
per_channel_quantization=per_channel_quantization,
6775
atol=0.25,
6876
qtol=1,
6977
)
@@ -72,15 +80,16 @@ def test_mv2_u55_BI():
7280

7381
@pytest.mark.slow
7482
@common.XfailIfNoCorstone320
75-
def test_mv2_u85_BI():
83+
@common.parametrize("per_channel_quantization", quant_test_data)
84+
def test_mv2_u85_BI(per_channel_quantization):
7685
pipeline = EthosU85PipelineBI[input_t](
7786
mv2,
7887
model_inputs,
7988
aten_ops=[],
8089
exir_ops=[],
8190
run_on_fvp=True,
8291
use_to_edge_transform_and_lower=True,
83-
per_channel_quantization=True,
92+
per_channel_quantization=per_channel_quantization,
8493
atol=0.25,
8594
qtol=1,
8695
)

‎backends/arm/test/ops/test_conv1d.py

Lines changed: 32 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -249,7 +249,7 @@ def forward(self, x):
249249
batches=1,
250250
)
251251

252-
test_modules = {
252+
test_data_MI = {
253253
"2_3x2x40_nobias": lambda: conv1d_2_3x2x40_nobias,
254254
"3_1x3x256_st1": lambda: conv1d_3_1x3x256_st1,
255255
"3_1x3x12_st2_pd1": lambda: conv1d_3_1x3x12_st2_pd1,
@@ -265,53 +265,65 @@ def forward(self, x):
265265
"two_conv1d": lambda: two_conv1d,
266266
}
267267

268+
test_data_BI = {
269+
f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q))
270+
for (k, v) in test_data_MI.items()
271+
for q in [True, False]
272+
}
273+
268274

269-
@common.parametrize("test_module", test_modules)
270-
def test_convolution_1d_tosa_MI(test_module):
275+
@common.parametrize("test_data", test_data_MI)
276+
def test_convolution_1d_tosa_MI(test_data):
271277
pipeline = TosaPipelineMI[input_t](
272-
test_module(),
273-
test_module().get_inputs(),
278+
test_data(),
279+
test_data().get_inputs(),
274280
aten_op,
275281
exir_op,
276282
)
277283
pipeline.run()
278284

279285

280-
@common.parametrize("test_module", test_modules)
281-
def test_convolution_1d_tosa_BI(test_module):
286+
@common.parametrize("test_data", test_data_BI)
287+
def test_convolution_1d_tosa_BI(test_data):
288+
model, per_channel_quantization = test_data()
282289
pipeline = TosaPipelineBI[input_t](
283-
test_module(),
284-
test_module().get_inputs(),
290+
model,
291+
model.get_inputs(),
285292
aten_op,
286293
exir_op,
294+
per_channel_quantization=per_channel_quantization,
295+
qtol=1,
287296
)
288-
pipeline.change_args("run_method_and_compare_outputs", qtol=1)
289297
pipeline.run()
290298

291299

292-
@common.parametrize("test_module", test_modules)
300+
@common.parametrize("test_data", test_data_BI)
293301
@common.XfailIfNoCorstone300
294-
def test_convolution_1d_u55_BI(test_module):
302+
def test_convolution_1d_u55_BI(test_data):
303+
model, per_channel_quantization = test_data()
295304
pipeline = EthosU55PipelineBI[input_t](
296-
test_module(),
297-
test_module().get_inputs(),
305+
model,
306+
model.get_inputs(),
298307
aten_op,
299308
exir_op,
300309
run_on_fvp=True,
310+
per_channel_quantization=per_channel_quantization,
311+
qtol=1,
301312
)
302-
pipeline.change_args("run_method_and_compare_outputs", qtol=1)
303313
pipeline.run()
304314

305315

306-
@common.parametrize("test_module", test_modules)
316+
@common.parametrize("test_data", test_data_BI)
307317
@common.XfailIfNoCorstone320
308-
def test_convolution_1d_u85_BI(test_module):
318+
def test_convolution_1d_u85_BI(test_data):
319+
model, per_channel_quantization = test_data()
309320
pipeline = EthosU85PipelineBI[input_t](
310-
test_module(),
311-
test_module().get_inputs(),
321+
model,
322+
model.get_inputs(),
312323
aten_op,
313324
exir_op,
314325
run_on_fvp=True,
326+
per_channel_quantization=per_channel_quantization,
327+
qtol=1,
315328
)
316-
pipeline.change_args("run_method_and_compare_outputs", qtol=1)
317329
pipeline.run()

‎backends/arm/test/ops/test_conv2d.py

Lines changed: 41 additions & 20 deletions
Original file line numberDiff line numberDiff line change
@@ -357,7 +357,7 @@ def forward(self, x):
357357

358358
# Shenanigan to get a nicer output when test fails. With unittest it looks like:
359359
# FAIL: test_convolution_2d_tosa_BI_2_3x3_1x3x12x12_st2_pd1
360-
test_modules = {
360+
test_data_MI = {
361361
"2x2_3x2x40x40_nobias": lambda: conv2d_2x2_3x2x40x40_nobias,
362362
"3x3_1x3x256x256_st1": lambda: conv2d_3x3_1x3x256x256_st1,
363363
"3x3_1x3x12x12_st2_pd1": lambda: conv2d_3x3_1x3x12x12_st2_pd1,
@@ -380,58 +380,79 @@ def forward(self, x):
380380
"groups_bias": lambda: conv2d_groups_bias,
381381
}
382382

383+
# Generate a new test set paired with per_channel_quant=True/False.
384+
test_data_BI = {
385+
f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q))
386+
for (k, v) in test_data_MI.items()
387+
for q in [True, False]
388+
# TODO: Invalid TOSA graph (MLETORCH-1144)
389+
if (k not in ["groups", "groups_bias"]) and (q is True)
390+
}
391+
383392
fvp_xfails = {
384-
"2x2_3x2x40x40_nobias": "MLETORCH-520: Numerical issues on FVP.",
385-
"5x5_3x2x128x128_st1": "MLETORCH-520: Numerical issues on FVP.",
393+
f"{k},per_channel_quant={q}": reason
394+
for k, reason in {
395+
"2x2_3x2x40x40_nobias": "MLETORCH-520: Numerical issues on FVP.",
396+
"5x5_3x2x128x128_st1": "MLETORCH-520: Numerical issues on FVP.",
397+
}.items()
398+
for q in [True, False]
386399
}
400+
387401
input_t = Tuple[torch.Tensor]
388402

389403

390-
@common.parametrize("test_module", test_modules)
391-
def test_convolution_2d_tosa_MI(test_module):
404+
@common.parametrize("test_data", test_data_MI)
405+
def test_convolution_2d_tosa_MI(test_data):
406+
model = test_data()
392407
pipeline = TosaPipelineMI[input_t](
393-
test_module(),
394-
test_module().get_inputs(),
408+
model,
409+
model.get_inputs(),
395410
aten_op,
396411
exir_op,
397412
)
398413
pipeline.run()
399414

400415

401-
@common.parametrize("test_module", test_modules)
402-
def test_convolution_2d_tosa_BI(test_module):
416+
@common.parametrize("test_data", test_data_BI)
417+
def test_convolution_2d_tosa_BI(test_data):
418+
model, per_channel_quantization = test_data()
403419
pipeline = TosaPipelineBI[input_t](
404-
test_module(),
405-
test_module().get_inputs(),
420+
model,
421+
model.get_inputs(),
406422
aten_op,
407423
exir_op,
424+
per_channel_quantization=per_channel_quantization,
425+
qtol=1,
408426
)
409-
pipeline.change_args("run_method_and_compare_outputs", qtol=1)
410427
pipeline.run()
411428

412429

413-
@common.parametrize("test_module", test_modules, fvp_xfails)
430+
@common.parametrize("test_data", test_data_BI, fvp_xfails)
414431
@common.XfailIfNoCorstone300
415-
def test_convolution_2d_u55_BI(test_module):
432+
def test_convolution_2d_u55_BI(test_data):
433+
model, per_channel_quantization = test_data()
416434
pipeline = EthosU55PipelineBI[input_t](
417-
test_module(),
418-
test_module().get_inputs(),
435+
model,
436+
model.get_inputs(),
419437
aten_op,
420438
exir_op,
421439
run_on_fvp=True,
440+
per_channel_quantization=per_channel_quantization,
422441
)
423442
pipeline.run()
424443

425444

426-
@common.parametrize("test_module", test_modules, fvp_xfails)
445+
@common.parametrize("test_data", test_data_BI, fvp_xfails)
427446
@common.XfailIfNoCorstone320
428-
def test_convolution_2d_u85_BI(test_module):
447+
def test_convolution_u85_BI(test_data):
448+
model, per_channel_quantization = test_data()
429449
pipeline = EthosU85PipelineBI[input_t](
430-
test_module(),
431-
test_module().get_inputs(),
450+
model,
451+
model.get_inputs(),
432452
aten_op,
433453
exir_op,
434454
run_on_fvp=True,
455+
per_channel_quantization=per_channel_quantization,
435456
)
436457
pipeline.run()
437458

‎backends/arm/test/ops/test_conv3d.py

Lines changed: 32 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -304,7 +304,7 @@ def forward(self, x):
304304
batches=1,
305305
)
306306

307-
test_modules = {
307+
test_data_MI = {
308308
"2x2_3x2x40x40_nobias": lambda: conv3d_2x2_3x2x40x40_nobias,
309309
"3x3_1x3x256x256_st1": lambda: conv3d_3x3_1x3x256x256_st1,
310310
"3x3_1x3x12x12_st2_pd1": lambda: conv3d_3x3_1x3x12x12_st2_pd1,
@@ -323,50 +323,66 @@ def forward(self, x):
323323
"3x3_1x3x224x224_st2_pd1": lambda: conv3d_3x3_1x3x224x224_st2_pd1,
324324
}
325325

326+
# Generate a new test set paired with per_channel_quant=True/False.
327+
test_data_BI = {
328+
f"{k},per_channel_quant={q}": (lambda v=v, q=q: (v(), q))
329+
for (k, v) in test_data_MI.items()
330+
for q in [True, False]
331+
}
332+
326333
input_t = Tuple[torch.Tensor]
327334

328335

329-
@common.parametrize("test_module", test_modules)
336+
@common.parametrize("test_data", test_data_MI)
330337
@pytest.mark.skip # Not implemented, skip until it is.
331-
def test_convolution_tosa_MI_3d(test_module):
338+
def test_convolution_3d_tosa_MI(test_data):
332339
pipeline = TosaPipelineMI[input_t](
333-
test_module(), test_module().get_inputs(), aten_op, exir_op
340+
test_data(), test_data().get_inputs(), aten_op, exir_op
334341
)
335342
pipeline.run()
336343

337344

338-
@common.parametrize("test_module", test_modules)
345+
@common.parametrize("test_data", test_data_BI)
339346
@pytest.mark.skip # Not implemented, skip until it is.
340-
def test_convolution_tosa_BI_3d(test_module):
347+
def test_convolution_3d_tosa_BI(test_data):
348+
model, per_channel_quantization = test_data()
341349
pipeline = TosaPipelineBI[input_t](
342-
test_module(), test_module().get_inputs(), aten_op, exir_op
350+
model,
351+
model.get_inputs(),
352+
aten_op,
353+
exir_op,
354+
per_channel_quantization=per_channel_quantization,
355+
qtol=1,
343356
)
344-
pipeline.change_args("run_method_and_compare_outputs", qtol=1)
345357
pipeline.run()
346358

347359

348-
@common.parametrize("test_module", test_modules)
360+
@common.parametrize("test_data", test_data_BI)
349361
@pytest.mark.skip # Not implemented, skip until it is.
350-
def test_convolution_u55_BI_3d(test_module):
362+
def test_convolution_3d_u55_BI(test_data):
363+
model, per_channel_quantization = test_data()
351364
pipeline = EthosU55PipelineBI[input_t](
352-
test_module(),
353-
test_module().get_inputs(),
365+
model,
366+
model.get_inputs(),
354367
aten_op,
355368
exir_op,
356369
run_on_fvp=True,
370+
per_channel_quantization=per_channel_quantization,
357371
)
358372
pipeline.run()
359373

360374

361-
@common.parametrize("test_module", test_modules)
375+
@common.parametrize("test_data", test_data_BI)
362376
@pytest.mark.skip # Not implemented, skip until it is.
363-
def test_convolution_u85_BI_3d(test_module):
377+
def test_convolution_3d_u85_BI(test_data):
378+
model, per_channel_quantization = test_data()
364379
pipeline = EthosU85PipelineBI[input_t](
365-
test_module(),
366-
test_module().get_inputs(),
380+
model,
381+
model.get_inputs(),
367382
aten_op,
368383
exir_op,
369384
run_on_fvp=True,
385+
per_channel_quantization=per_channel_quantization,
370386
)
371387
pipeline.run()
372388

0 commit comments

Comments
 (0)