From 80c163186e1e69c42c60f88452c387817af22f91 Mon Sep 17 00:00:00 2001 From: Jinu Sunil Date: Tue, 24 Sep 2024 16:33:10 -0700 Subject: [PATCH 1/6] Update pytorch_tabular_benchmark.py Add benchmark results to code comments. --- benchmark/pytorch_tabular_benchmark.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/benchmark/pytorch_tabular_benchmark.py b/benchmark/pytorch_tabular_benchmark.py index 5631af21..c1c558e6 100644 --- a/benchmark/pytorch_tabular_benchmark.py +++ b/benchmark/pytorch_tabular_benchmark.py @@ -1,5 +1,17 @@ """This script benchmarks the training time of TabTransformer using PyTorch Frame and PyTorch Tabular. + +Results form comparing Pytorch Tabular and Frame. Specifically the iteration +speed while trainig. + +------------------------------------- +Package | Model | Num iters/sec| +------------------------------------- +Tabular | TabNet | 41.7 +Frame | TabNet | 45.0 +Tabular | FTTrans | 40.1 +Frame | FTTrans | 43.7 +-------------------------------------- """ import argparse import os.path as osp From bbb60ca96cdca113be973236089c321522592c1d Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 24 Sep 2024 23:33:30 +0000 Subject: [PATCH 2/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- benchmark/pytorch_tabular_benchmark.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/benchmark/pytorch_tabular_benchmark.py b/benchmark/pytorch_tabular_benchmark.py index c1c558e6..9540424d 100644 --- a/benchmark/pytorch_tabular_benchmark.py +++ b/benchmark/pytorch_tabular_benchmark.py @@ -7,7 +7,7 @@ ------------------------------------- Package | Model | Num iters/sec| ------------------------------------- -Tabular | TabNet | 41.7 +Tabular | TabNet | 41.7 Frame | TabNet | 45.0 Tabular | FTTrans | 40.1 Frame | FTTrans | 43.7 From d48382df5af83597e6b567781594e11262ca1882 Mon Sep 17 00:00:00 2001 From: Jinu Sunil Date: Mon, 30 Sep 2024 14:16:24 -0700 Subject: [PATCH 3/6] Update README.md --- benchmark/README.md | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/benchmark/README.md b/benchmark/README.md index 3264dff5..feadb6b5 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -209,3 +209,16 @@ Experimental setting: 20 Optuna search trials for XGBoost, CatBoost and LightGBM | FTTransformer | 0.872±0.005 (7004s) | 0.540±0.068 (3355s) | 0.908±0.004 (7514s) | | TabNet | **0.912±0.004 (219s)** | 0.995±0.001 (301s) | 0.919±0.003 (187s) | | TabTransformer | 0.843±0.003 (2810s) | 0.657±0.187 (2843s) | 0.854±0.001 (284s) | + +## Benchmarking pytorch-frame and pytorch-tabular + +`pytorch_tabular_benchmark` compares `ptroch-frame` to `pytorch-tabular`. `pytorch-tabular` is a popular package for Deep learning(DL) on tabular data. Its key use case it to make using existing tabular DL models easy. `pytorch_tabular_benchmar` primarly compares runtimes for similar models across the two libraries. + +| Package | Model | Num iters/sec| +| :-------| :-------|:-------------| +| Tabular | TabNet | 41.7 | +| Frame | TabNet | 45.0 | +| Tabular | FTTrans | 40.1 | +| Frame | FTTrans | 43.7 | + + From 700da76a302b8cc0a6783aa14ce978d607b41466 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 30 Sep 2024 21:16:41 +0000 Subject: [PATCH 4/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- benchmark/README.md | 14 ++++++-------- 1 file changed, 6 insertions(+), 8 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index feadb6b5..6728df51 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -214,11 +214,9 @@ Experimental setting: 20 Optuna search trials for XGBoost, CatBoost and LightGBM `pytorch_tabular_benchmark` compares `ptroch-frame` to `pytorch-tabular`. `pytorch-tabular` is a popular package for Deep learning(DL) on tabular data. Its key use case it to make using existing tabular DL models easy. `pytorch_tabular_benchmar` primarly compares runtimes for similar models across the two libraries. -| Package | Model | Num iters/sec| -| :-------| :-------|:-------------| -| Tabular | TabNet | 41.7 | -| Frame | TabNet | 45.0 | -| Tabular | FTTrans | 40.1 | -| Frame | FTTrans | 43.7 | - - +| Package | Model | Num iters/sec | +| :------ | :------ | :------------ | +| Tabular | TabNet | 41.7 | +| Frame | TabNet | 45.0 | +| Tabular | FTTrans | 40.1 | +| Frame | FTTrans | 43.7 | From 463631eef5e54d359208e4eca300bb3311f1b106 Mon Sep 17 00:00:00 2001 From: Yiwen Yuan Date: Wed, 2 Oct 2024 11:32:47 -0700 Subject: [PATCH 5/6] Update README.md --- benchmark/README.md | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index 6728df51..fdd58e79 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -212,11 +212,12 @@ Experimental setting: 20 Optuna search trials for XGBoost, CatBoost and LightGBM ## Benchmarking pytorch-frame and pytorch-tabular -`pytorch_tabular_benchmark` compares `ptroch-frame` to `pytorch-tabular`. `pytorch-tabular` is a popular package for Deep learning(DL) on tabular data. Its key use case it to make using existing tabular DL models easy. `pytorch_tabular_benchmar` primarly compares runtimes for similar models across the two libraries. +`pytorch_tabular_benchmark` compares the performance of `pytorch-frame` to `pytorch-tabular`. `pytorch-tabular` excels in providing an accessible approach for standard tabular tasks, allowing users to quickly implement and experiment with existing tabular learning models. It also excels with its training loop modifications and explainability feature. On the other hand, `ptroch-frame` offers enhanced flexibility for exploring and building novel tabular learning approaches while still providing access to established models. It distinguishes itself through support for a wider array of data types, more sophisticated encoding schemas, and streamlined integration with LLMs. +The following table shows the speed comparison of `pytorch-frame` to `pytorch-tabular` on implementations of `TabNet` and `FTTransformer`. | Package | Model | Num iters/sec | | :------ | :------ | :------------ | -| Tabular | TabNet | 41.7 | -| Frame | TabNet | 45.0 | -| Tabular | FTTrans | 40.1 | -| Frame | FTTrans | 43.7 | +| PyTorch Tabular | TabNet | 41.7 | +| PyTorch Frame | TabNet | 45.0 | +| PyTorch Tabular | FTTransformer | 40.1 | +| PyTorch Frame | FTTransformer | 43.7 | From dcd7220b0db544a377b941d2d06b9276d587b03a Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 2 Oct 2024 18:33:08 +0000 Subject: [PATCH 6/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- benchmark/README.md | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/benchmark/README.md b/benchmark/README.md index fdd58e79..afaee57b 100644 --- a/benchmark/README.md +++ b/benchmark/README.md @@ -215,9 +215,9 @@ Experimental setting: 20 Optuna search trials for XGBoost, CatBoost and LightGBM `pytorch_tabular_benchmark` compares the performance of `pytorch-frame` to `pytorch-tabular`. `pytorch-tabular` excels in providing an accessible approach for standard tabular tasks, allowing users to quickly implement and experiment with existing tabular learning models. It also excels with its training loop modifications and explainability feature. On the other hand, `ptroch-frame` offers enhanced flexibility for exploring and building novel tabular learning approaches while still providing access to established models. It distinguishes itself through support for a wider array of data types, more sophisticated encoding schemas, and streamlined integration with LLMs. The following table shows the speed comparison of `pytorch-frame` to `pytorch-tabular` on implementations of `TabNet` and `FTTransformer`. -| Package | Model | Num iters/sec | -| :------ | :------ | :------------ | -| PyTorch Tabular | TabNet | 41.7 | -| PyTorch Frame | TabNet | 45.0 | +| Package | Model | Num iters/sec | +| :-------------- | :------------ | :------------ | +| PyTorch Tabular | TabNet | 41.7 | +| PyTorch Frame | TabNet | 45.0 | | PyTorch Tabular | FTTransformer | 40.1 | | PyTorch Frame | FTTransformer | 43.7 |