Merge branch 'release/v1.1.0'

Author: rsnitsch

.gitattributes

@@ -0,0 +1 @@
+*.torrent filter=lfs diff=lfs merge=lfs -text

.gitignore

@@ -1,9 +1,12 @@
+__pycache__
 docs/build/
-src/__pycache__/
 Pipfile.lock
 dist
 src/py3createtorrent.egg-info
 build
 testdata/
 .idea/
-*.torrent
+benchmark/*.csv
+benchmark/*.png
+benchmark/py3createtorrent.py
+benchmark/results

src/.style.yapf renamed to .style.yapf

@@ -5,8 +5,8 @@ python setup.py bdist_wheel
 
 # Uploading
 
-twine upload dist/*
-twine upload --repository testpypi dist/*
+twine upload --skip-existing dist/*
+twine upload --skip-existing --repository testpypi dist/*
 
 # Testing
 

BUILD.md

@@ -8,6 +8,10 @@ sphinx = "~=3.1"
 mypy = "~=0.782"
 yapf = "~=0.30"
 twine = "*"
+matplotlib = "*"
+pandas = "*"
+torf-cli = {file = "https://github.com/rndusr/torf-cli/archive/refs/heads/master.zip"}
+faker = "*"
 
 [packages]
 "bencode.py" = "~=4.0"

Pipfile

@@ -10,16 +10,15 @@ Features
 
 Some of the features:
 
-* you can create **huge torrents** for any amount of data
-* you can add a **comment** to the torrent file
-* you can create **private torrents** (disabled DHT, ...)
-* you can create torrents with **multiple trackers**
-* you can create **trackerless torrents**
-* you can add **webseeds** to torrents
-* you can **exclude specific files/folders**
-* you can exclude files/folders based on **regular expressions**
-* you can specify **custom piece sizes**
-* you can specify custom creation dates
+* create torrents with **multiple trackers** or **trackerless torrents**
+* **automatically choose the most reliable trackers** from [ngosang/trackerslist](https://github.com/ngosang/trackerslist)
+* fast torrent creation thanks to **multithreading**
+* add **webseeds** to torrents
+* create **private torrents** (disabled DHT, ...)
+* **exclude specific files/folders**
+* exclude files/folders based on **regular expressions**
+* specify **custom piece sizes**
+* specify custom creation dates
 
 Basic usage
 -----------

README.md

@@ -0,0 +1,13 @@
+FROM debian:stable
+
+WORKDIR /benchmark
+
+RUN apt-get update
+RUN apt-get install -y mktorrent transmission-cli python3 python3-pip wget
+
+RUN wget https://github.com/sharkdp/hyperfine/releases/download/v1.13.0/hyperfine_1.13.0_amd64.deb
+RUN dpkg -i hyperfine_1.13.0_amd64.deb
+
+RUN pip3 install py3createtorrent torf-cli matplotlib pandas
+
+COPY py3createtorrent.py benchmark.sh create_random_file.py create_random_folder.py plot_benchmark_results.py ./

benchmark/Dockerfile

@@ -0,0 +1,13 @@
+set size=15g
+set target=../tests/testdata/random_file_%size%.dat
+set warmup=1
+set runs=3
+set threads=1,4
+
+python create_random_file.py ../tests/testdata/random_file_%size%.dat %size%
+
+hyperfine --warmup %warmup% --runs %runs% --export-csv benchmark_results.csv -L threads %threads% -L piece_size 128,1024,8192 "python ../src/py3createtorrent.py %target% -p {piece_size} --threads {threads}" "torrenttools create %target% -v1 --piece-size {piece_size}K --threads {threads}"
+
+hyperfine --warmup %warmup% --runs %runs% --export-csv benchmark_results_torf.csv -L threads %threads% -L piece_size 0.125,1,8 --show-output "torf %target% --yes --threads {threads} --max-piece-size {piece_size}"
+
+python plot_benchmark_results.py benchmark_results.csv benchmark_results_torf.csv

benchmark/benchmark.bat

@@ -0,0 +1,16 @@
+#!/bin/sh
+target=random_file_4gib.dat
+python3 create_random_file.py $target 1g
+
+warmup=1
+runs=3
+threads=1,2,3,4
+
+hyperfine --warmup $warmup --runs $runs --export-csv /results/benchmark_results.csv -L threads $threads -L piece_size 128,1024,8192 --show-output "python3 py3createtorrent.py $target -p {piece_size} --threads {threads}" "transmission-create --piece-size {piece_size} $target"
+
+hyperfine --warmup $warmup --runs $runs --export-csv /results/benchmark_results_torf.csv -L threads $threads -L piece_size 0.125,1,8 --show-output "torf $target --yes --threads {threads} --max-piece-size {piece_size}"
+
+hyperfine --warmup $warmup --runs $runs --prepare "rm *.torrent" --export-csv /results/benchmark_results_mktorrent.csv -L threads $threads -L piece_size 17,20,23 --show-output "mktorrent -t{threads} -l{piece_size} $target"
+
+cd /results
+python3 /benchmark/plot_benchmark_results.py benchmark_results.csv benchmark_results_torf.csv benchmark_results_mktorrent.csv

benchmark/benchmark.sh

@@ -0,0 +1,4 @@
+copy /Y ..\src\py3createtorrent.py .
+docker build . --tag benchmark
+docker container rm benchmark
+docker container run --name benchmark -v %cd%\results:/results benchmark sh /benchmark/benchmark.sh

benchmark/benchmark_docker.bat

@@ -0,0 +1,66 @@
+"""
+Script to create files of given size with pseudo-random content.
+"""
+import argparse
+import os
+import random
+import sys
+
+
+def parse_size(size):
+    letter_to_unit = {"K": 2**10, "M": 2**20, "G": 2**30}
+
+    error_message = ("must be a number ending with a single letter to "
+                     "indicate the unit type K, M or G - for example, use 256k to specify 256 KiB")
+
+    letter = size[-1]
+    if letter.isdigit():
+        raise argparse.ArgumentTypeError(error_message)
+
+    number = size[:-1]
+    if not number.isdigit():
+        raise argparse.ArgumentTypeError(error_message)
+
+    return letter_to_unit[letter.upper()] * int(number)
+
+
+def create_random_file(dst, size):
+    BLOCK_SIZE = 4096
+    bytes_saved = 0
+    with open(dst, "wb") as fh:
+        while True:
+            remaining_bytes = size - bytes_saved
+            if remaining_bytes:
+                data = random.randbytes(min(remaining_bytes, BLOCK_SIZE))
+                bytes_saved += len(data)
+                fh.write(data)
+            else:
+                break
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", help="destination file path")
+    parser.add_argument(
+        "size",
+        type=parse_size,
+        help="size in KiB/MiB/GiB, specify unit with a single suffix letter K/M/G, for example 256k for 256 KiB")
+    parser.add_argument("--seed", type=int, default=0, help="Set seed for the random number generator.")
+    parser.add_argument("--overwrite", action="store_true", help="Overwrite existing file.")
+
+    args = parser.parse_args()
+
+    if os.path.isfile(args.path) and not args.overwrite:
+        if os.path.getsize(args.path) != args.size:
+            print("ERROR: Destination file already exists BUT DOES NOT HAVE THE CORRECT SIZE", file=sys.stderr)
+        else:
+            print("WARNING: Destination file already exists (already has the requested size)", file=sys.stderr)
+        print("Not doing anything. Use --overwrite option to force overwriting the existing file.")
+        sys.exit(1)
+
+    random.seed(args.seed)
+    create_random_file(args.path, args.size)
+
+
+if __name__ == '__main__':
+    main()

benchmark/create_random_file.py

@@ -0,0 +1,65 @@
+"""
+Script to generate a specified number of files with pseudo-random names/contents in
+a given directory. The size of each file will be randomly (uniformly) chosen from a
+user-specified range.
+"""
+import argparse
+import random
+from pathlib import Path
+
+from create_random_file import parse_size, create_random_file
+from faker import Faker
+
+
+def create_cache_dir_tag(path):
+    path = Path(path)
+    if path.is_dir():
+        path = path.joinpath("CACHEDIR.TAG")
+    elif not path.name == "CACHEDIR.TAG":
+        raise ValueError("Full file path was specified, but file name is not CACHEDIR.TAG")
+
+    with open(path, "w") as fh:
+        fh.write("Signature: 8a477f597d28d172789f06886806bc55\n")
+        fh.write("# This file instructs backup applications to ignore this directory.\n")
+        fh.write("# For more information see https://www.brynosaurus.com/cachedir/\n")
+
+    return path
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("path", type=Path, help="destination folder path")
+    parser.add_argument("number_of_files", type=int, help="Number of files to generate")
+    parser.add_argument(
+        "min_file_size",
+        type=parse_size,
+        help=
+        "minimum file size in KiB/MiB/GiB, specify unit with a single suffix letter K/M/G, for example 256k for 256 KiB"
+    )
+    parser.add_argument("max_file_size", type=parse_size, help="maximum file size")
+    parser.add_argument("--seed", type=int, default=0, help="Set seed for the random number generator.")
+    parser.add_argument("--no-cachedir-tag", action="store_true", help="Do not generate CACHEDIR.TAG file")
+
+    args = parser.parse_args()
+
+    if args.min_file_size > args.max_file_size:
+        parser.error("min_file_size must be smaller or equal than max_file_size")
+
+    random.seed(args.seed)
+    fake = Faker()
+    Faker.seed(args.seed)
+
+    args.path.mkdir(parents=True, exist_ok=True)
+    if not args.no_cachedir_tag:
+        p = create_cache_dir_tag(args.path)
+        print("Saved cachedir tag at: %s" % p)
+
+    for i in range(args.number_of_files):
+        filename = fake.file_name()
+        size = random.randint(args.min_file_size, args.max_file_size)
+        print("Creating random file: % 20s of size: % 10d bytes..." % (filename, size))
+        create_random_file(args.path.joinpath(filename), size)
+
+
+if __name__ == '__main__':
+    main()

benchmark/create_random_folder.py

@@ -0,0 +1,77 @@
+import argparse
+import os
+
+import matplotlib.pyplot as plt
+import pandas as pd
+
+
+def generate_plot_for_piece_size(df, piece_size):
+    df = df[df["parameter_piece_size"] == piece_size].copy()
+    df = df.drop(columns="parameter_piece_size")
+
+    TOOLS = ["py3createtorrent", "torrenttools", "torf", "mktorrent", "transmission-create"]
+
+    for tool in TOOLS:
+        df.loc[df["command"].str.contains(tool), "tool"] = tool
+
+    df = df.drop(columns="command")
+    df = df.set_index(["parameter_threads", "tool"])
+
+    tools = df.index.unique(level=1)
+    threads = df.index.unique(level=0)
+    max_threads = max(threads)
+
+    fig, ax = plt.subplots(1, 1, figsize=(8, 6))
+
+    for idx, tool in enumerate(tools):
+        mean = df.loc[df.index.get_level_values(1) == tool, "mean"]
+        stddev = df.loc[df.index.get_level_values(1) == tool, "stddev"]
+        ax.plot(threads, mean)
+        #ax.errorbar(threads, mean, stddev, linestyle='None', marker='x')
+
+    fig.suptitle("Performance (lower = faster)", fontsize=20)
+    ax.set_title("Piece size = %d KiB" % piece_size)
+    ax.set_xlabel("Number of threads for hashing")
+    ax.set_ylabel("Time in s")
+    ax.set_xticks(list(range(1, max_threads + 1)))
+    ax.legend(tools, loc='upper right')
+
+    fig.savefig("plot_for_piece_size_%dk.png" % piece_size, dpi=125)
+    plt.close(fig)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("results_file", help="path to CSV file with the results", nargs='+')
+
+    args = parser.parse_args()
+
+    for file in args.results_file:
+        if not os.path.isfile(file):
+            parser.error("The specified results file does not exist: " + file)
+
+    df = None
+    for file in args.results_file:
+        df_file = pd.read_csv(file)
+        if df is None:
+            df = df_file
+        else:
+            df = pd.concat([df, df_file])
+
+    # Normalize piece sizes
+    #print(df[df["command"].str.contains("torf")].head())
+    df.loc[df["command"].str.contains("torf"), "parameter_piece_size"] *= 2**10
+    df.loc[df["command"].str.contains("mktorrent"), "parameter_piece_size"] = 2**(df.loc[df["command"].str.contains("mktorrent"), "parameter_piece_size"] - 10)
+
+    #print(df)
+    piece_sizes = df["parameter_piece_size"].unique()
+
+    plt.style.use('ggplot')
+    for p in piece_sizes:
+        print("Generating plot for piece size %s" % p)
+        generate_plot_for_piece_size(df.copy(), p)
+        print()
+
+
+if __name__ == '__main__':
+    main()

benchmark/plot_benchmark_results.py

@@ -1,6 +1,15 @@
 Changelog
 =========
 
+Version 1.1.0
+-------------
+
+*Release date: 2022/03/18*
+
+* added: **multithreading** for much faster torrent creation (about 30-40% faster). The number of threads can be controlled with the new ``--threads`` option. It defaults to using 4 threads which will be a good choice on most systems.
+* added: ``--version`` command
+* improved: formatting and content of ``--help`` output
+
 Version 1.0.1
 -------------
 

docs/source/changelog.rst

@@ -48,9 +48,9 @@
 # built documents.
 #
 # The short X.Y version.
-version = '1.0'
+version = '1.1'
 # The full version, including alpha/beta/rc tags.
-release = '1.0.1'
+release = '1.1.0'
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.