Add HTTP upload support for VOD

Signed-off-by: Mingyang Wu <mingyangx.wu@intel.com>

Author: mingwu123123

cdn-server/Dockerfile

@@ -52,6 +52,7 @@ RUN mkdir -p /home/build/var/www/tmp/client_body && \
     mkdir -p /home/build/var/www/tmp/uwsgi && \
     mkdir -p /home/build/var/www/tmp/scgi && \
     mkdir -p /home/build/var/www/cache && \
+    mkdir -p /home/build/var/www/temp && \
     mkdir -p /home/build/var/www/html
 
 
@@ -62,7 +63,7 @@ WORKDIR /home
 
 # Prerequisites
 RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime && \
-    DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends libxml2 libssl-dev libpcre3 zlib1g libxslt1.1 python3-tornado python3-kafka python3-bs4 python3-kazoo vim openssh-server && \
+    DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends libxml2 libssl-dev libpcre3 zlib1g libxslt1.1 python3-tornado python3-kafka python3-bs4 python3-kazoo vim openssh-server ffmpeg && \
     rm -rf /var/lib/apt/lists/*
 # Install
 COPY --from=build /home/build /

cdn-server/config.ini

@@ -4,4 +4,5 @@ timeSpan=20000
 srcMode=local
 [path]
 srcPath=/var/www/archive
+tempPath=/var/www/temp
 

cdn-server/main.py

@@ -4,12 +4,21 @@
 from tornado.options import define, options, parse_command_line
 from playlist import PlayListHandler
 from schedule import ScheduleHandler
+from upload import UploadHandler
+import os
+import configparser
 
 APP = web.Application([
     (r'/playlist', PlayListHandler),
     (r'/schedule/.*', ScheduleHandler),
+    (r'/upload/', UploadHandler),
 ])
 
+config = configparser.ConfigParser()
+config.read('config.ini')
+tempPath = config.get('path', 'tempPath')
+srcPath = config.get('path', 'srcPath')
+
 if __name__ == "__main__":
     define("port", default=2222, help="the binding port", type=int)
     define("ip", default="127.0.0.1", help="the binding ip")

cdn-server/nginx.conf

@@ -89,6 +89,12 @@ http {
             proxy_pass http://localhost:2222;
         }
 
+        location /upload/ {
+            add_header Cache-Control no-cache;
+            proxy_pass http://localhost:2222;
+            client_max_body_size 200M;
+        }
+
         location /hls/ {
             root /var/www;
             add_header Cache-Control no-cache;

cdn-server/upload.py

@@ -0,0 +1,24 @@
+import os
+import shutil
+from tornado.web import RequestHandler
+
+class UploadHandler(RequestHandler):
+    def post(self, *args, **kwargs):
+        fileName = self.get_body_argument('fileName', None)
+        file = self.request.files.get('file', None)
+        uploadStatus = self.get_body_argument('uploadStatus', None)
+        timeStamp = self.get_body_argument('timeStamp', None)
+        from main import tempPath,srcPath
+        fileName = timeStamp + "-" + fileName
+        proPath = os.path.join(tempPath, fileName)
+        try:
+            with open(proPath, 'ab') as f:
+                f.write(file[0]['body'])
+                self.set_status(200)
+            if uploadStatus == 'end':
+                shutil.move(proPath, srcPath)
+                cmd = "ffmpeg -i " + srcPath + "/" + fileName + " -vf thumbnail,scale=640:360 -frames:v 1 -y " + srcPath + "/" + fileName + ".png"
+                os.system(cmd)		
+        except Exception as e:
+            print(e)
+        return

deployment/kubernetes/yaml_utils.py

@@ -47,7 +47,7 @@ def add_volumeMounts(data, isCDN):
     if (isCDN):
         volumemounts_caps = [ {'name': 'archive',
                                'mountPath': '/var/www/archive',
-                               'readOnly': True},
+                               'readOnly': False},
                               {'name': 'dash',
                                'mountPath': '/var/www/dash',
                                'readOnly': False},

script/install_dependency.sh

@@ -72,6 +72,7 @@ elif [ "$LINUX_DISTRO" == "CentOS" ]; then
     try_command yum install -y epel-release 
     try_command yum install -y python36 python36-pip
     try_command pip3 install ruamel.yaml
+    try_command pip3 install pytest-shutil
 else
     echo -e $ECHO_PREFIX_INFO "The installation will be cancelled."
     echo -e $ECHO_PREFIX_INFO "The CDN-Transcode-Sample does not support this OS, please use Ubuntu 18.04 or CentOS 7.6.\n"

volume/html/css/app.css

@@ -20,3 +20,30 @@ form .form-icons .fa {
   color: white;
   width: 1rem;
 }
+
+.input-group-0 {
+  display: none;
+}
+
+.flex-center{
+  width: 100%;
+  height: 50px;
+  display: none;
+}
+
+.flex-center .input-group-bar {
+    border-style: solid;
+    border-width: 1px;
+    border-color: #cccccc;
+    width: 80%;
+    height: 10px;
+    border-radius: 2px;
+    margin: 0 auto;
+}
+.flex-center .bar{
+    background-color: #ff4c4c;
+    width: 0%;
+    height: 100%;
+
+}
+

volume/html/header.shtml

@@ -11,6 +11,7 @@
       <li><a href="#" user-name-menu class="menu-text">guest</a>
         <ul class="menu" style="background-color:gray">
           <li><a href="#" data-open="setting" class="menu-text"><i class="fi-key"></i>&nbsp;Setting</a></li>
+          <li><a href="#" data-open="upload" class="menu-text"><i class="fi-key"></i>&nbsp;Upload</a></li>
         </ul>
       </li>
     </ul>
@@ -29,3 +30,28 @@
     <input class="button expanded" type="submit" data-close>
   </form>
 </div>
+
+<div class="reveal" id="upload" data-reveal>
+  <form>
+    <p><h4><b>Upload</b></h4></p>
+
+    <div class="input-group-0" ui-header-setting-user>
+      <input class="input_file" type="file" name="file" required="required"/>
+    </div>
+
+    <div class="input-group choose-file" ui-header-setting-user>
+      <span class="input-group-label" style="width:40%"><b>Choose File:</b></span>
+      <input class="input-group-field" type="text" name="user" readonly required placeholder="Please choose mp4 file"><br>
+    </div>
+
+
+    <div class="flex-center">
+        <h6 style="text-align:center">Upload 0% </h6>
+        <div class="input-group-bar">
+            <div class="bar"></div>
+        </div>
+    </div>
+
+    <input class="button expanded" type="button" value="Submit"  vadata-close>
+  </form>
+</div>

volume/html/js/app-header.js

@@ -21,3 +21,86 @@ var settings={
         return typeof localStorage.user!="undefined"?localStorage.user:"guest";
     },
 }
+
+
+//upload
+$("#upload .choose-file").click(
+    function() {
+        $("#upload .input_file").trigger('click')
+    }
+)
+
+$("#upload .input_file").change(
+    function() {
+        var filePath = $("#upload .input_file").val()
+        var pos=filePath.lastIndexOf("\\");
+        var fileName = filePath.substring(pos+1);
+        $("#upload .choose-file .input-group-field").val(fileName)
+    }
+)
+
+$("#upload").find("form .button").click(function() {
+    if(!$("#upload .input_file").val()){
+	return false
+    }  
+    if (!(document.querySelector("#upload .input_file").files[0].name.endsWith('.mp4'))) {
+        $(".flex-center").show()
+        $(".flex-center h6").html("Please choose mp4 file")
+        $(".flex-center .input-group-bar").hide()
+        return false
+    } else {
+        upload()
+    }
+});
+
+function upload() {
+    $(".flex-center").show()
+    $(".flex-center .input-group-bar").show()
+    const LENGTH = 1024 * 1024 * 10;
+    var timeStamp = new Date().getTime();
+    var fileName=$("#upload .choose-file .input-group-field").val();
+    var file = document.querySelector('#upload .input_file').files[0];
+    var totalSize = file.size;
+    var start = 0;
+    var end = start + LENGTH;
+    var fd = null;
+    var blob = null;
+    var xhr = null;
+    var sum = Math.ceil(totalSize/LENGTH)
+    var count = 0
+    var timer = setInterval(function () {
+        if (start < totalSize) {
+            fd = new FormData();
+            xhr = new XMLHttpRequest();
+            xhr.open('POST', '/upload/', false);
+            blob = file.slice(start, end);
+            fd.append('file', blob);
+            fd.append('fileName', fileName);
+            fd.append('timeStamp', timeStamp);
+            if (end >= totalSize) {
+                fd.append('uploadStatus','end')
+            }
+            xhr.send(fd);
+            if (xhr.status != 200 && xhr.status != 0) {
+                $(".flex-center h6").html("Error, Please try again")
+                $(".flex-center .input-group-bar").hide()
+                return false
+            }
+            count += 1
+            $(".flex-center h6").html("Upload " + parseInt(count * 100 / sum) + "%")
+            $(".flex-center .bar").width(parseInt(count * 100 / sum) + "%")
+            start = end;
+            end = start + LENGTH;
+        } else {
+            $(".flex-center h6").html("Upload 0%")
+            $(".flex-center .bar").width("0%")
+            $(".flex-center").hide()
+            $("#upload .input_file").val('')
+            $("#upload .choose-file .input-group-field").val('')
+            $("#setting").find("form").submit();
+            $(".reveal-overlay").trigger('click');
+            clearInterval(timer);
+        }
+    },100)
+}
+