Update and improve tool scripts

lukasmasuch · lukasmasuch · commit 889d0f3bc4bd · 2021-01-10T19:45:25.000Z
diff --git a/resources/tools/filebrowser.sh b/resources/tools/filebrowser.sh
@@ -18,7 +18,8 @@ if [ ! -f "/usr/local/bin/filebrowser"  ]; then
     echo "Installing Filebrowser. Please wait..."
     mkdir -p $RESOURCES_PATH/filebrowser
     cd $RESOURCES_PATH/filebrowser
-    wget -q https://github.com/filebrowser/filebrowser/releases/download/v2.10.0/linux-amd64-filebrowser.tar.gz -O ./filebrowser.tar.gz
+    FILEBROWSER_VERSION=2.11.0
+    wget -q https://github.com/filebrowser/filebrowser/releases/download/v$FILEBROWSER_VERSION/linux-amd64-filebrowser.tar.gz -O ./filebrowser.tar.gz
     tar -xzf ./filebrowser.tar.gz
     chmod +x "./filebrowser"
     mv "./filebrowser" "/usr/local/bin/filebrowser"
diff --git a/resources/tools/flink-local-cluster.sh b/resources/tools/flink-local-cluster.sh
@@ -22,7 +22,8 @@ if [ ! -d "$FLINK_HOME" ]; then
     cd $RESOURCES_PATH
     FLINK_VERSION=1.12.0
     SCALA_VERSION=2.12
-    wget https://ftp.fau.de/apache/flink/flink-$FLINK_VERSION/flink-$FLINK_VERSION-bin-scala_$SCALA_VERSION.tgz -O ./flink.tar.gz
+    echo "Downloading. Please wait..."
+    wget -q https://ftp.fau.de/apache/flink/flink-$FLINK_VERSION/flink-$FLINK_VERSION-bin-scala_$SCALA_VERSION.tgz -O ./flink.tar.gz
     tar xzf flink.tar.gz
     mv flink-$FLINK_VERSION $FLINK_HOME
     rm flink.tar.gz
diff --git a/resources/tools/gpuview.sh b/resources/tools/gpuview.sh
@@ -17,7 +17,7 @@ done
 if ! hash gpuview 2>/dev/null; then
     echo "Installing GPUview. Please wait..."
     # https://github.com/fgaim/gpuview
-    pip install --no-cache-dir gpuview
+    pipx install gpuview
 else
     echo "GPUview is already installed"
 fi
diff --git a/resources/tools/hadoop-local-cluster.sh b/resources/tools/hadoop-local-cluster.sh
@@ -34,7 +34,8 @@ if [ ! -d "$HADOOP_HOME" ]; then
     echo "Installing Hadoop. Please wait..."
     cd $RESOURCES_PATH
     HADOOP_VERSION=3.3.0
-    wget https://apache.mirror.digionline.de/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz -O ./hadoop.tar.gz
+    echo "Downloading..."
+    wget -q https://apache.mirror.digionline.de/hadoop/common/hadoop-$HADOOP_VERSION/hadoop-$HADOOP_VERSION.tar.gz -O ./hadoop.tar.gz
     tar xzf hadoop.tar.gz
     mv hadoop-$HADOOP_VERSION $HADOOP_HOME
     rm hadoop.tar.gz
diff --git a/resources/tools/label-studio.sh b/resources/tools/label-studio.sh
@@ -0,0 +1,36 @@
+#!/bin/sh
+
+# Stops script execution if a command has an error
+set -e
+
+INSTALL_ONLY=0
+PORT=""
+# Loop through arguments and process them: https://pretzelhands.com/posts/command-line-flags
+for arg in "$@"; do
+    case $arg in
+        -i|--install) INSTALL_ONLY=1 ; shift ;;
+        -p=*|--port=*) PORT="${arg#*=}" ; shift ;; # TODO Does not allow --port 1234
+        *) break ;;
+    esac
+done
+
+if ! hash label-studio 2>/dev/null; then
+    echo "Installing Label Studio. Please wait..."
+    pipx install label-studio
+else
+    echo "Label Studio is already installed"
+fi
+
+# Run
+if [ $INSTALL_ONLY = 0 ] ; then
+    if [ -z "$PORT" ]; then
+        read -p "Please provide a port for starting Label Studio: " PORT
+    fi
+
+    echo "Starting Label Studio on port "$PORT
+    cd $WORKSPACE_HOME
+    # Create tool entry for tooling plugin
+    echo '{"id": "label-studio-link", "name": "Label Studio", "url_path": "/tools/'$PORT'/import", "description": "Multi-type data labeling & annotation tool"}' > $HOME/.workspace/tools/label-studio.json
+    label-studio start labeling_project --allow-serving-local-files --init -p $PORT --no-browser
+    sleep 15
+fi
diff --git a/resources/tools/netron.sh b/resources/tools/netron.sh
@@ -17,7 +17,7 @@ done
 if ! hash netron 2>/dev/null; then
     echo "Installing Netron. Please wait..."
     # https://github.com/lutzroeder/netron
-    pip install --no-cache-dir  netron
+    pipx install netron
 else
     echo "Netron is already installed"
 fi
diff --git a/resources/tools/oh-my-zsh.sh b/resources/tools/oh-my-zsh.sh
@@ -27,7 +27,8 @@ if ! hash zsh 2>/dev/null; then
     rm -rf fonts
     # Install plugins
     apt-get install -y --no-install-recommends autojump git-flow git-extras ncdu htop
-    pip install Pygments ranger-fm thefuck
+    pip install Pygments ranger-fm thefuck bpytop
+    # Install fkill-cli: (too big - 30MB) npm install --global fkill-cli && \
     yes | sh -c "$(curl -fsSL https://raw.github.com/robbyrussell/oh-my-zsh/master/tools/install.sh)"
     # Install powerlevel10k for instant prompt
     # git clone --depth=1 https://github.com/romkatv/powerlevel10k.git ${ZSH_CUSTOM:-$HOME/.oh-my-zsh/custom}/themes/powerlevel10k
diff --git a/resources/tools/portainer.sh b/resources/tools/portainer.sh
@@ -17,7 +17,7 @@ done
 if [ ! -f "$RESOURCES_PATH/portainer/portainer"  ]; then
     echo "Installing Portainer. Please wait..."
     cd $RESOURCES_PATH
-    PORTAINER_VERSION=2.0.0
+    PORTAINER_VERSION=2.0.1
     wget https://github.com/portainer/portainer/releases/download/$PORTAINER_VERSION/portainer-$PORTAINER_VERSION-linux-amd64.tar.gz
     tar xvpfz portainer-$PORTAINER_VERSION-linux-amd64.tar.gz
     rm ./portainer-$PORTAINER_VERSION-linux-amd64.tar.gz
diff --git a/resources/tools/scala-utils.sh b/resources/tools/scala-utils.sh
@@ -40,6 +40,8 @@ else
 fi
 
 
+# TODO: Install Coursier https://get-coursier.io/docs/cli-installation
+
 if [ ! -d "$HOME/.local/share/jupyter/kernels/scala" ]; then
     echo "Installing Almond Scala Kernel for Jupyter. Please wait..."
     curl -Lo coursier https://git.io/coursier-cli
diff --git a/resources/tools/spark-local-cluster.sh b/resources/tools/spark-local-cluster.sh
@@ -15,18 +15,33 @@ done
 # Script inspired by: https://github.com/jupyter/docker-stacks/blob/master/pyspark-notebook/Dockerfile#L18
 # https://github.com/apache/incubator-toree/blob/master/Dockerfile
 
+
+# Todo: Add additional spark configuration:
+# https://spark.apache.org/docs/latest/configuration.html
+# TODO start spark master?
+# https://medium.com/@marcovillarreal_40011/creating-a-spark-standalone-cluster-with-docker-and-docker-compose-ba9d743a157f
+# ENV SPARK_MASTER_PORT 7077
+# ENV SPARK_MASTER_WEBUI_PORT 8080
+# ENV SPARK_WORKER_WEBUI_PORT 8081
+# ENV SPARK_MASTER_LOG /spark/logs
+# ENV SPARK_WORKER_LOG /spark/logs
+# export SPARK_MASTER_HOST=`hostname`
+# SPARK_WORKER_CORES=1
+# SPARK_WORKER_MEMORY=1G
+# SPARK_DRIVER_MEMORY=128m
+# SPARK_EXECUTOR_MEMORY=256m
+
+# TODO configure spark ui to be proxied with base path:
+# https://stackoverflow.com/questions/45971127/wrong-css-location-of-spark-application-ui
+# https://github.com/jupyterhub/jupyter-server-proxy/issues/57
+# https://github.com/yuvipanda/jupyter-sparkui-proxy/blob/master/jupyter_sparkui_proxy/__init__.py
+
+
 # Install scala 2.12
 if [[ ! $(scala -version 2>&1) =~ "version 2.12" ]]; then
     # Update to Scala 2.12 is required for spark
-    SCALA_VERSION=2.12.12
-    echo "Updating to Scala $SCALA_VERSION. Please wait..."
-    apt-get remove scala-library scala
-    apt-get autoremove
-    wget -q https://downloads.lightbend.com/scala/$SCALA_VERSION/scala-$SCALA_VERSION.deb -O ./scala.deb
-    dpkg -i scala.deb
-    rm scala.deb
-    apt-get update
-    apt-get install scala
+    echo "Scala 2.12 is not installed. You should consider running the scala-utils.sh tool installer before continuing."
+    sleep 10
 else
     echo "Scala 2.12 already installed."
 fi
@@ -38,7 +53,8 @@ if [ ! -d "$SPARK_HOME" ]; then
     cd $RESOURCES_PATH
     SPARK_VERSION="3.0.1"
     HADOOP_VERSION="3.2"
-    wget https://mirror.checkdomain.de/apache/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz -O ./spark.tar.gz
+    echo "Downloading. Please wait..."
+    wget -q https://mirror.checkdomain.de/apache/spark/spark-$SPARK_VERSION/spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION.tgz -O ./spark.tar.gz
     tar xzf spark.tar.gz
     mv spark-$SPARK_VERSION-bin-hadoop$HADOOP_VERSION/ $SPARK_HOME
     rm spark.tar.gz
@@ -55,12 +71,12 @@ if [ ! -d "$SPARK_HOME" ]; then
     pip install --no-cache-dir sparkmagic
     jupyter serverextension enable --py sparkmagic
 
-    # Install sparkmonitor: https://github.com/krishnan-r/sparkmonitor
-    pip install --no-cache-dir sparkmonitor
-    jupyter nbextension install sparkmonitor --py --sys-prefix --symlink
-    jupyter nbextension enable sparkmonitor --py --sys-prefix
-    jupyter serverextension enable --py --sys-prefix sparkmonitor
-    ipython profile create && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >>  $(ipython profile locate default)/ipython_kernel_config.py
+    # TODO: does not work right now: Install sparkmonitor: https://github.com/krishnan-r/sparkmonitor
+    # pip install --no-cache-dir sparkmonitor
+    # jupyter nbextension install sparkmonitor --py --sys-prefix --symlink
+    # jupyter nbextension enable sparkmonitor --py --sys-prefix
+    # jupyter serverextension enable --py --sys-prefix sparkmonitor
+    # ipython profile create && echo "c.InteractiveShellApp.extensions.append('sparkmonitor.kernelextension')" >>  $(ipython profile locate default)/ipython_kernel_config.py
 
     # Deprecated: jupyter-spark: https://github.com/mozilla/jupyter-spark
     # jupyter serverextension enable --py jupyter_spark && \
@@ -85,8 +101,10 @@ pip install --no-cache-dir pyspark findspark pyarrow spylon-kernel
 if [[ ! $(jupyter kernelspec list) =~ "toree" ]]; then
     echo "Installing Toree Kernel for Jupyter. Please wait..."
     TOREE_VERSION=0.5.0
-    pip install --no-cache-dir https://dist.apache.org/repos/dist/dev/incubator/toree/$TOREE_VERSION-incubating-rc1/toree-pip/toree-$TOREE_VERSION.tar.gz
-    jupyter toree install --sys-prefix --spark_home=$SPARK_HOME
+    echo "Torre Kernel does not seem to work with the installed spark and scala verison."
+    # TODO: Fix installation
+    # pip install --no-cache-dir https://dist.apache.org/repos/dist/dev/incubator/toree/$TOREE_VERSION-incubating-rc1/toree-pip/toree-$TOREE_VERSION.tar.gz
+    # jupyter toree install --sys-prefix --spark_home=$SPARK_HOME
 else
     echo "Toree Kernel for Jupyter is already installed."
 fi
diff --git a/resources/tools/ungit.sh b/resources/tools/ungit.sh
@@ -17,7 +17,7 @@ done
 if ! hash ungit 2>/dev/null; then
     echo "Installing Ungit. Please wait..."
     npm update
-    npm install -g ungit@1.5.14
+    npm install -g ungit@1.5.15
 else
     echo "Ungit is already installed"
 fi
diff --git a/resources/tools/vs-code-server.sh b/resources/tools/vs-code-server.sh
@@ -17,7 +17,7 @@ done
 if [ ! -f "/usr/local/bin/code-server"  ]; then
     echo "Installing VS Code Server. Please wait..."
     cd ${RESOURCES_PATH}
-    VS_CODE_VERSION=3.7.4
+    VS_CODE_VERSION=3.8.0
     # Use yarn install since it is smaller
     yarn --production --frozen-lockfile global add code-server@"$VS_CODE_VERSION"
     yarn cache clean
diff --git a/resources/tools/zeppelin.sh b/resources/tools/zeppelin.sh
@@ -23,7 +23,7 @@ if [ ! -f "$RESOURCES_PATH/zeppelin/zeppelin-$ZEPPELIN_VERSION-bin-all/bin/zeppe
     mkdir ./zeppelin
     cd ./zeppelin
     echo "Downloading. Please wait..."
-    wget https://www.apache.org/dist/zeppelin/zeppelin-$ZEPPELIN_VERSION/zeppelin-$ZEPPELIN_VERSION-bin-all.tgz -O ./zeppelin-$ZEPPELIN_VERSION-bin-all.tgz
+    wget -q https://www.apache.org/dist/zeppelin/zeppelin-$ZEPPELIN_VERSION/zeppelin-$ZEPPELIN_VERSION-bin-all.tgz -O ./zeppelin-$ZEPPELIN_VERSION-bin-all.tgz
     tar xfz zeppelin-$ZEPPELIN_VERSION-bin-all.tgz
     rm zeppelin-$ZEPPELIN_VERSION-bin-all.tgz
     # https://github.com/mirkoprescha/spark-zeppelin-docker/blob/master/Dockerfile#L40
