Custom Dataset construction

[abdullah-al-masud]

Hi there, I followed your documentation and tried to build non in-memory dataset. I have already prepared Data() objects for each graph separately inside the 'raw' directory and trying to just use the dataloader to load mini-batches. Can you please guide me how to do that ?

Currently I am using this piece of code to do that.

class MyOwnDataset(Dataset):
    def __init__(self, root, transform=None, pre_transform=None, pre_filter=None):
        super().__init__(root, transform, pre_transform, pre_filter)

    @property
    def raw_file_names(self):
        return sorted(os.listdir(self.raw_dir))

    @property
    def processed_file_names(self):
        return sorted(os.listdir(self.raw_dir))

    def process(self):
        idx = 0
        for raw_path in self.raw_paths:
            # Read data from `raw_path`.

            if self.pre_filter is not None and not self.pre_filter(data):
                continue

            if self.pre_transform is not None:
                data = self.pre_transform(data)
            shutil.copyfile(os.path.join(raw_path), os.path.join(self.processed_dir, os.path.split(raw_path)[-1]))

    def len(self):
        return len(self.processed_file_names)

    def get(self, idx):
        data = torch.load(os.path.join(self.processed_dir, self.processed_file_names[idx]))
        return data

But I am getting this error while extracting batch data using for loop

Traceback (most recent call last):
File "/home/masud/workspace/thesis/src/operate/train.py", line 32, in
run_training()
File "/home/masud/workspace/thesis/src/operate/train.py", line 27, in run_training
tmodel.fit(train_loader=train_loader, val_loader=val_loader, evaluate=False)
File "/home/masud/workspace/thesis/./src/models/trainer.py", line 49, in fit
for i, batch_data in enumerate(train_loader):
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 633, in next
data = self._next_data()
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/utils/data/dataloader.py", line 677, in _next_data
data = self._dataset_fetcher.fetch(index) # may raise StopIteration
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch/utils/data/_utils/fetch.py", line 54, in fetch
return self.collate_fn(data)
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch_geometric/loader/dataloader.py", line 20, in call
return Batch.from_data_list(batch, self.follow_batch,
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch_geometric/data/batch.py", line 76, in from_data_list
batch, slice_dict, inc_dict = collate(
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch_geometric/data/collate.py", line 85, in collate
value, slices, incs = _collate(attr, values, data_list, stores,
File "/home/masud/anaconda3/envs/thesis/lib/python3.9/site-packages/torch_geometric/data/collate.py", line 160, in _collate
value = torch.cat(values, dim=cat_dim or 0, out=out)
RuntimeError: Sizes of tensors must match except in dimension 0. Expected size 22 but got size 25 for tensor number 1 in the list.

[wsad1]

It looks like for some graph attribute the shape isn't same across all graphs. Can you check the shape of attributes for each graph , attributes like x, edge_index, edge_attr etc?

[abdullah-al-masud]

Yes, I found the issue. Actually I prepared the edge_attr features incorrectly. It requires to be of dimension (number_of_edges, feature_dimension) but I prepared them in (number_of_nodes, number_of_nodes, feature_dimension). After modifying the edge_attr tensor, it works now.