- read the docs baseline

Author: giventocode

args.go

@@ -25,8 +25,8 @@ const (
 	s3SecretKeyEnvVar = "S3_SECRET_KEY"
 )
 
-//defautls for some of the arguments and parameters...
-const defaulPreSignedExpMins = 90
+//defaults for some of the arguments and parameters...
+const defaulPreSignedExpMins = 360
 const numOfWorkersFactor = 8
 const numOfReadersFactor = 5
 const defaultNumberOfFilesInBatch = 500

docs/Makefile

@@ -0,0 +1,20 @@
+# Minimal makefile for Sphinx documentation
+#
+
+# You can set these variables from the command line.
+SPHINXOPTS    =
+SPHINXBUILD   = sphinx-build
+SPHINXPROJ    = BlobPorter
+SOURCEDIR     = .
+BUILDDIR      = _build
+
+# Put it first so that "make" without argument is like "make help".
+help:
+	@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
+
+.PHONY: help Makefile
+
+# Catch-all target: route all unknown targets to Sphinx using the new
+# "make mode" option.  $(O) is meant as a shortcut for $(SPHINXOPTS).
+%: Makefile
+	@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)

docs/conf.py

@@ -0,0 +1,155 @@
+# -*- coding: utf-8 -*-
+#
+# Configuration file for the Sphinx documentation builder.
+#
+# This file does only contain a selection of the most common options. For a
+# full list see the documentation:
+# http://www.sphinx-doc.org/en/stable/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+# import os
+# import sys
+# sys.path.insert(0, os.path.abspath('.'))
+
+
+# -- Project information -----------------------------------------------------
+
+project = u'BlobPorter'
+#copyright = u'2018, Jesus Aguilar'
+#author = u'Jesus Aguilar'
+
+# The short X.Y version
+version = u''
+# The full version, including alpha/beta/rc tags
+release = u''
+
+
+# -- General configuration ---------------------------------------------------
+
+# If your documentation needs a minimal Sphinx version, state it here.
+#
+# needs_sphinx = '1.0'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = [
+]
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = ['_templates']
+
+# The suffix(es) of source filenames.
+# You can specify multiple suffix as a list of string:
+#
+# source_suffix = ['.rst', '.md']
+source_suffix = '.rst'
+
+# The master toctree document.
+master_doc = 'index'
+
+# The language for content autogenerated by Sphinx. Refer to documentation
+# for a list of supported languages.
+#
+# This is also used if you do content translation via gettext catalogs.
+# Usually you set "language" from the command line for these cases.
+language = None
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path .
+exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+
+# The name of the Pygments (syntax highlighting) style to use.
+pygments_style = 'sphinx'
+
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Theme options are theme-specific and customize the look and feel of a theme
+# further.  For a list of options available for each theme, see the
+# documentation.
+#
+# html_theme_options = {}
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = ['_static']
+
+# Custom sidebar templates, must be a dictionary that maps document names
+# to template names.
+#
+# The default sidebars (for documents that don't match any pattern) are
+# defined by theme itself.  Builtin themes are using these templates by
+# default: ``['localtoc.html', 'relations.html', 'sourcelink.html',
+# 'searchbox.html']``.
+#
+# html_sidebars = {}
+
+
+# -- Options for HTMLHelp output ---------------------------------------------
+
+# Output file base name for HTML help builder.
+htmlhelp_basename = 'BlobPorterdoc'
+
+
+# -- Options for LaTeX output ------------------------------------------------
+
+latex_elements = {
+    # The paper size ('letterpaper' or 'a4paper').
+    #
+    # 'papersize': 'letterpaper',
+
+    # The font size ('10pt', '11pt' or '12pt').
+    #
+    # 'pointsize': '10pt',
+
+    # Additional stuff for the LaTeX preamble.
+    #
+    # 'preamble': '',
+
+    # Latex figure (float) alignment
+    #
+    # 'figure_align': 'htbp',
+}
+
+# Grouping the document tree into LaTeX files. List of tuples
+# (source start file, target name, title,
+#  author, documentclass [howto, manual, or own class]).
+latex_documents = [
+    (master_doc, 'BlobPorter.tex', u'BlobPorter Documentation',
+     u'Jesus Aguilar', 'manual'),
+]
+
+
+# -- Options for manual page output ------------------------------------------
+
+# One entry per manual page. List of tuples
+# (source start file, name, description, authors, manual section).
+man_pages = [
+    (master_doc, 'blobporter', u'BlobPorter Documentation',
+     [author], 1)
+]
+
+
+# -- Options for Texinfo output ----------------------------------------------
+
+# Grouping the document tree into Texinfo files. List of tuples
+# (source start file, target name, title, author,
+#  dir menu entry, description, category)
+texinfo_documents = [
+    (master_doc, 'BlobPorter', u'BlobPorter Documentation',
+     author, 'BlobPorter', 'One line description of project.',
+     'Miscellaneous'),
+]

docs/index.rst

@@ -0,0 +1,20 @@
+.. BlobPorter documentation master file, created by
+   sphinx-quickstart on Mon Feb 12 13:52:41 2018.
+   You can adapt this file completely to your liking, but it should at least
+   contain the root `toctree` directive.
+
+Welcome to BlobPorter's documentation!
+======================================
+
+.. toctree::
+   :maxdepth: 2
+   :caption: Contents:
+
+
+
+Indices and tables
+==================
+
+* :ref:`genindex`
+* :ref:`modindex`
+* :ref:`search`

docs/make.bat

@@ -0,0 +1,36 @@
+@ECHO OFF
+
+pushd %~dp0
+
+REM Command file for Sphinx documentation
+
+if "%SPHINXBUILD%" == "" (
+	set SPHINXBUILD=sphinx-build
+)
+set SOURCEDIR=.
+set BUILDDIR=_build
+set SPHINXPROJ=BlobPorter
+
+if "%1" == "" goto help
+
+%SPHINXBUILD% >NUL 2>NUL
+if errorlevel 9009 (
+	echo.
+	echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
+	echo.installed, then set the SPHINXBUILD environment variable to point
+	echo.to the full path of the 'sphinx-build' executable. Alternatively you
+	echo.may add the Sphinx directory to PATH.
+	echo.
+	echo.If you don't have Sphinx installed, grab it from
+	echo.http://sphinx-doc.org/
+	exit /b 1
+)
+
+%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+goto end
+
+:help
+%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
+
+:end
+popd

docs/perfmode.rst

@@ -0,0 +1,42 @@
+Performance Mode
+======================================
+
+If you want to maximize performance, and your source and target are public HTTP based end-points (Blob, S3, and HTTP), running the transfer in a high bandwidth environment such as a VM on the cloud, is strongly recommended.  This recommendation comes from the fact that blob to blob, S3 to blob or HTTP to blob transfers are bidirectional where BlobPorter downloads the data (without writing to disk) and uploads it as it is received. 
+
+When running in the cloud, consider the region where the transfer VM ( where BlobPorter will be running), will be deployed. When possible, deploy the transfer VM in the same the same region as the target of the transfer.  Running in the same region as the target minimizes the transfer costs (egress from the VM to the target storage account) and the network performance impact (lower latency) for the upload operation.
+
+For downloads or uploads of multiple or large files the disk i/o could be the constraining resource that slows down the transfer. And often identifying if this is the case, is a cumbersome process.  But if done, it could lead to informed decisions about the environment where BlobPorter runs.
+
+To help with this indentification process, BlobPorter has a performance mode that uploads random data generated in memory and measures the performance of the operation without the impact of disk i/o.
+The performance mode for uploads could help you identify the potential upper limit of throughput that the network and the target storage account can provide.   
+
+For example, the following command will upload 10 x 10GB files to a storage account.
+
+```
+blobporter -f "1GB:10" -c perft -t perf-blockblob
+```
+
+You can also use this mode to see if increasing (or decreasing) the number of workers/writers (-g option) will have a potential impact.
+
+```
+blobporter -f "1GB:10" -c perft -t perf-blockblob -g 20
+```
+
+Similarly, for downloads, you can simulate downloading data from a storage account without writing to disk. This mode could also help you fine-tune the number of readers (-r option) and get an idea of the maximum download throughput.
+
+The following command will download the data we previously uploaded.
+
+```
+export SRC_ACCOUNT_KEY=$ACCOUNT_KEY
+blobporter -f "https://myaccount.blob.core.windows.net/perft" -t blob-perf 
+```
+
+Then you can try downloading to disk.
+
+```
+blobporter -c perft -t blob-file 
+```
+
+If the performance difference is significant then you can conclude that disk i/o is the bottleneck, at which point you can consider an SSD backed VM.
+
+

sources/multifile.go

@@ -52,7 +52,7 @@ type FileSystemSourceParams struct {
 
 // NewFileSystemSourcePipeline creates a new MultiFilePipeline.
 // If the sourcePattern results in a single file and the targetAlias is set, the alias will be used as the target name.
-// Otherwise the full original file name will be used.
+// Otherwise the original file name will be used.
 func NewFileSystemSourcePipeline(params *FileSystemSourceParams) []pipeline.SourcePipeline {
 	var files []string
 	var err error

sources/ostorefactory.go

@@ -10,8 +10,8 @@ import (
 //AzureBlobSource constructs parts channel and implements data readers for Azure Blobs exposed via HTTP
 type AzureBlobSource struct {
 	HTTPSource
-	Container      string
-	BlobNames      []string
+	container      string
+	blobNames      []string
 	exactNameMatch bool
 }
 
@@ -26,8 +26,8 @@ func NewAzureBlobSourcePipeline(params *AzureBlobParams) []pipeline.SourcePipeli
 	}
 
 	factory := func(httpSource HTTPSource) (pipeline.SourcePipeline, error) {
-		return &AzureBlobSource{Container: params.Container,
-			BlobNames:      params.BlobNames,
+		return &AzureBlobSource{container: params.Container,
+			blobNames:      params.BlobNames,
 			HTTPSource:     httpSource,
 			exactNameMatch: params.SourceParams.UseExactNameMatch}, nil
 	}