First release

Signed-off-by: Zhou, Cheng <cheng.zhou@intel.com>

Author: czhou26

CMakeLists.txt

@@ -0,0 +1,14 @@
+cmake_minimum_required (VERSION 2.8)
+
+Project(OVC NONE)
+
+file(GLOB dirs "deployment" "*")
+list(REMOVE_DUPLICATES dirs)
+foreach(dir ${dirs})
+    if(EXISTS ${dir}/CMakeLists.txt)
+        add_subdirectory(${dir})
+    endif()
+endforeach()
+
+# legal message
+execute_process(COMMAND printf "\nThis script will build third party components licensed under various open source licenses into your container images. The terms under which those components may be used and distributed can be found with the license document that is provided with those components. Please familiarize yourself with those terms to ensure your distribution of those components complies with the terms of those licenses.\n\n")

LICENSE

@@ -0,0 +1,14 @@
+This project is for Open Visual Cloud CDN Transcode E2E Sample.
+
+## Key ingredients in this repo
+- Dockerfiles
+- CMakefiles
+- Configuration files
+- Helper scripts
+- Reference Architecture document
+- User Guide (Getting Started Guide) document
+
+## How to get started?
+- Refer to "doc/CDN_Transcode_Sample_RA.md" to understand the design.
+- Refer to "doc/CDN_Transcode_Sample_Getting_Started_Guide.md" to setup and run the sample.
+

README.md

@@ -0,0 +1,4 @@
+CMakeLists.txt
+*.sh
+*.m4
+test/*

cdn-server/.dockerignore

@@ -0,0 +1,2 @@
+set(service "ovc_cdn_service")
+include("${CMAKE_SOURCE_DIR}/script/service.cmake")

cdn-server/CMakeLists.txt

@@ -0,0 +1,73 @@
+
+FROM ubuntu:18.04 AS build
+WORKDIR /home
+
+# COMMON BUILD TOOLS
+RUN DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends build-essential autoconf make git wget pciutils cpio libtool lsb-release ca-certificates pkg-config bison flex
+
+# Install cmake
+ARG CMAKE_VER=3.13.1
+ARG CMAKE_REPO=https://cmake.org/files
+RUN wget -O - ${CMAKE_REPO}/v${CMAKE_VER%.*}/cmake-${CMAKE_VER}.tar.gz | tar xz && \
+    cd cmake-${CMAKE_VER} && \
+    ./bootstrap --prefix="/usr" && \
+    make -j8 && \
+    make install
+
+# Install automake, use version 1.14 on CentOS
+ARG AUTOMAKE_VER=1.14
+ARG AUTOMAKE_REPO=https://ftp.gnu.org/pub/gnu/automake/automake-${AUTOMAKE_VER}.tar.xz
+    RUN apt-get install -y -q automake
+
+# Build nginx-rtmp
+ARG NGINX_RTMP_VER=v1.2.1
+ARG NGINX_RTMP_REPO=https://github.com/arut/nginx-rtmp-module/archive/${NGINX_RTMP_VER}.tar.gz
+ARG NGINX_RTMP_PATCH_HLS=https://raw.githubusercontent.com/VCDP/CDN/master/0001-add-hevc-support-for-rtmp-and-hls.patch
+ARG NGINX_RTMP_PATCH_DASH=https://raw.githubusercontent.com/VCDP/CDN/master/0002-add-HEVC-support-for-dash.patch
+
+RUN wget -O - ${NGINX_RTMP_REPO} | tar xz && mv nginx-rtmp-module-${NGINX_RTMP_VER#v} nginx-rtmp-module && \
+    cd nginx-rtmp-module && \
+    wget -O - ${NGINX_RTMP_PATCH_HLS} | patch -p1 && \
+    wget -O - ${NGINX_RTMP_PATCH_DASH} | patch -p1
+
+
+# Build nginx & nginx-rtmp
+ARG NGINX_VER=1.14.2
+ARG NGINX_REPO=https://nginx.org/download/nginx-${NGINX_VER}.tar.gz
+
+RUN apt-get update && apt-get install -y -q --no-install-recommends libssl-dev libpcre3-dev zlib1g-dev libxslt1-dev
+
+RUN wget -O - ${NGINX_REPO} | tar xz && \
+    cd nginx-${NGINX_VER} && \
+    ./configure --prefix=/var/www --sbin-path=/usr/sbin/nginx --modules-path=/usr/lib/x86_64-linux-gnu/nginx/modules --conf-path=/etc/nginx/nginx.conf --error-log-path=/var/www/log/error.log --pid-path=/var/www/nginx.pid --lock-path=/var/www/nginx.lock --http-log-path=/var/www/log/access.log --http-client-body-temp-path=/var/www/tmp/client_body --http-proxy-temp-path=/var/www/tmp/proxy --http-fastcgi-temp-path=/var/www/tmp/fastcgi --http-uwsgi-temp-path=/var/www/tmp/uwsgi --http-scgi-temp-path=/var/www/tmp/scgi --user=www-data --group=www-data --with-select_module --with-poll_module --with-threads --with-file-aio --with-http_ssl_module --with-http_v2_module --with-http_realip_module --with-http_addition_module --with-http_xslt_module --with-http_sub_module --with-http_dav_module --with-http_flv_module --with-http_mp4_module --with-http_gunzip_module --with-http_gzip_static_module --with-http_auth_request_module --with-http_random_index_module --with-http_secure_link_module --with-http_degradation_module --with-http_slice_module --with-http_stub_status_module --with-stream --with-stream_ssl_module --with-stream_realip_module --with-stream_ssl_preread_module --with-pcre --add-module=../nginx-rtmp-module && \
+    make -j8 && \
+    make install DESTDIR=/home/build
+
+# NGINX Setup
+RUN mkdir -p /home/build/var/www/tmp/client_body && \
+    mkdir -p /home/build/var/www/tmp/proxy && \
+    mkdir -p /home/build/var/www/tmp/fastcgi && \
+    mkdir -p /home/build/var/www/tmp/uwsgi && \
+    mkdir -p /home/build/var/www/tmp/scgi && \
+    mkdir -p /home/build/var/www/cache && \
+    mkdir -p /home/build/var/www/html
+
+
+FROM ubuntu:18.04
+LABEL Description="This is the base image for a NGINX+RTMP service"
+LABEL Vendor="Intel Corporation"
+WORKDIR /home
+
+# Prerequisites
+RUN ln -sf /usr/share/zoneinfo/UTC /etc/localtime && \
+    DEBIAN_FRONTEND=noninteractive apt-get update && apt-get install -y -q --no-install-recommends libxml2 libssl-dev libpcre3 zlib1g libxslt1.1 python3-tornado python3-kafka python3-bs4 python3-kazoo vim && \
+    rm -rf /var/lib/apt/lists/*
+# Install
+COPY --from=build /home/build /
+COPY *.xsl  /etc/nginx/
+COPY *.conf /etc/nginx/
+COPY *.ini  /home/
+COPY *.py   /home/
+CMD  ["/bin/bash","-c","/home/main.py&/usr/sbin/nginx"]
+VOLUME /etc/nginx /var/www/html /var/www/tmp/client_body /var/www/tmp/proxy /var/www/tmp/fastcgi /var/www/tmp/uwsgi /var/www/tmp/scgi /var/www/cache /var/www/dash /var/www/hls
+EXPOSE 8080

cdn-server/Dockerfile

@@ -0,0 +1,6 @@
+#!/bin/bash -e
+
+IMAGE="ovc_cdn_service"
+DIR=$(dirname $(readlink -f "$0"))
+
+. "${DIR}/../script/build.sh"

cdn-server/build.sh

@@ -0,0 +1,7 @@
+[run]
+timeSpan=20000
+[mode]
+srcMode=local
+[path]
+srcPath=/var/www/archive
+

cdn-server/config.ini

@@ -0,0 +1,19 @@
+#!/usr/bin/python3
+
+from tornado import ioloop, web
+from tornado.options import define, options, parse_command_line
+from playlist import PlayListHandler
+from schedule import ScheduleHandler
+
+APP = web.Application([
+    (r'/playlist', PlayListHandler),
+    (r'/schedule/.*', ScheduleHandler),
+])
+
+if __name__ == "__main__":
+    define("port", default=2222, help="the binding port", type=int)
+    define("ip", default="127.0.0.1", help="the binding ip")
+    parse_command_line()
+    print("Listening to " + options.ip + ":" + str(options.port))
+    APP.listen(options.port, address=options.ip)
+    ioloop.IOLoop.instance().start()

cdn-server/main.py

@@ -0,0 +1,91 @@
+#!/usr/bin/python3
+
+import socket
+from kafka import KafkaProducer, KafkaConsumer, TopicPartition
+
+KAFKA_HOSTS = ["kafka:9092"]
+
+class Producer():
+    def __init__(self):
+        super(Producer, self).__init__()
+        self._client_id = socket.gethostname()
+        self._producer = None
+
+    def send(self, topic, message):
+        if not self._producer:
+            try:
+                self._producer = KafkaProducer(bootstrap_servers=KAFKA_HOSTS,
+                                               client_id=self._client_id,
+                                               api_version=(0, 10), retries=1)
+            except Exception as e:
+                print(str(e))
+                self._producer = None
+
+        if self._producer:
+            try:
+                self._producer.send(topic, message.encode('utf-8'))
+                print("send "+topic+": ")
+                print(message)
+            except Exception as e:
+                print(str(e))
+        else:
+            print("producer not available")
+
+    def flush(self):
+        if self._producer:
+            self._producer.flush()
+
+    def close(self):
+        if self._producer:
+            self.flush()
+            self._producer.close()
+
+class Consumer():
+    def __init__(self, group=None):
+        super(Consumer, self).__init__()
+        self._client_id = socket.gethostname()
+        self._group = group
+
+    def messages(self, topic, timeout=None):
+        c = KafkaConsumer(topic, bootstrap_servers=KAFKA_HOSTS, client_id=self._client_id,
+                          group_id=self._group, api_version=(0, 10))
+
+        partitions = c.partitions_for_topic(topic)
+        if not partitions:
+            raise Exception("Topic "+topic+" not exist")
+
+        timeout1 = 100 if timeout is None else timeout
+        while True:
+            partitions = c.poll(timeout1)
+            if partitions:
+                for p in partitions:
+                    for msg in partitions[p]:
+                        yield msg.value.decode('utf-8')
+            if timeout is not None:
+                yield ""
+
+        c.close()
+
+    def debug(self, topic):
+        c = KafkaConsumer(bootstrap_servers=KAFKA_HOSTS, client_id=self._client_id,
+                          group_id=None, api_version=(0, 10))
+
+        # assign/subscribe topic
+        partitions = c.partitions_for_topic(topic)
+        if not partitions:
+            raise Exception("Topic "+topic+" not exist")
+        c.assign([TopicPartition(topic, p) for p in partitions])
+
+        # seek to beginning if needed
+        c.seek_to_beginning()
+
+        # fetch messages
+        while True:
+            partitions = c.poll(100)
+            if partitions:
+                for p in partitions:
+                    for msg in partitions[p]:
+                        yield msg.value.decode('utf-8')
+            yield ""
+
+        c.close()

cdn-server/messaging.py

@@ -0,0 +1,151 @@
+
+worker_processes auto;
+worker_rlimit_nofile 8192;
+daemon off;
+
+events {
+    worker_connections  4096;
+}
+
+rtmp {
+    server {
+        listen 1935;
+        chunk_size 4000;
+
+        application stream {
+            live on;
+        }
+
+        application hls {
+            live on;
+            hls on;
+            hls_path /var/www/hls;
+            hls_nested on;
+            hls_fragment 3;
+            hls_playlist_length 60;
+        }
+
+        application dash {
+            live on;
+            dash on;
+            dash_path /var/www/dash;
+            dash_fragment 3;
+            dash_playlist_length 60;
+            dash_nested on;
+        }
+    }
+}
+
+http {
+    include mime.types;
+    default_type application/octet-stream;
+    proxy_cache_path /var/www/cache levels=1:2 keys_zone=one:10m use_temp_path=off;
+
+    server {
+        listen 8080 ssl;
+        server_name _;
+        client_body_timeout 5s;
+        client_header_timeout 5s;
+
+        ssl_certificate /run/secrets/self.crt;
+        ssl_certificate_key /run/secrets/self.key;
+        ssl_dhparam /run/secrets/dhparam.pem;
+
+        ssl_protocols TLSv1.2;
+        ssl_prefer_server_ciphers on;
+        ssl_ciphers "EECDH+AESGCM:EDH+AESGCM:AES256+EECDH:AES256+EDH";
+        ssl_ecdh_curve secp384r1;
+        ssl_session_cache shared:SSL:10m;
+        ssl_session_tickets off;
+        ssl_stapling off;
+        ssl_stapling_verify off;
+
+        location / {
+            root /var/www/html;
+            sendfile on;
+
+            # proxy cache settings
+            proxy_cache one;
+            proxy_no_cache $http_pragma $http_authorization;
+            proxy_cache_bypass $cookie_nocache $arg_nocache$arg_comment;
+            proxy_cache_valid 200 302 10m;
+            proxy_cache_valid 303 1m;
+
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header 'X-Frame-Options' 'deny' always;
+            add_header 'X-XSS-Protection' '1' always;
+            add_header 'X-Content-Type-Options' 'nosniff' always;
+            ssi on;
+        }
+
+        location /api/playlist {
+            add_header Cache-Control no-cache;
+            rewrite ^/api(/playlist.*) $1 break;
+            proxy_pass http://localhost:2222;
+        }
+
+        location /hls/ {
+            alias /var/www/hls/;
+            add_header Cache-Control no-cache;
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header 'Access-Control-Expose-Headers' 'Content-Length';
+            types {
+                application/vnd.apple.mpegurl m3u8;
+                video/mp2t ts;
+            }
+        }
+
+        location /dash/ {
+            alias /var/www/dash/;
+            add_header Cache-Control no-cache;
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header 'Access-Control-Expose-Headers' 'Content-Length';
+            types {
+                application/dash+xml mpd;
+            }
+        }
+
+        location ~* /dash/.*/index.mpd$ {
+            alias /var/www;
+            add_header Cache-Control no-cache;
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header 'Access-Control-Expose-Headers' 'Content-Length';
+            types {
+                application/dash+xml mpd;
+            }
+            try_files $uri @dashls;
+        }
+
+        location ~* /hls/.*/index.m3u8$ {
+            alias /var/www;
+            add_header Cache-Control no-cache;
+            add_header 'Access-Control-Allow-Origin' '*' always;
+            add_header 'Access-Control-Expose-Headers' 'Content-Length';
+            types {
+                application/vnd.apple.mpegurl m3u8;
+            }
+            try_files $uri @dashls;
+        }
+
+        location @dashls {
+            add_header Cache-Control no-cache;
+            rewrite ^/(dash|hls)/(.*) /schedule/$1/$2 break;
+            proxy_pass http://localhost:2222;
+        }
+
+        location ~* /thumbnail/.*.png$ {
+            root /var/www/archive;
+            add_header Cache-Control no-cache;
+            rewrite ^/thumbnail(/.*) $1 break;
+        }
+
+        location /stat {
+            rtmp_stat all;
+            rtmp_stat_stylesheet stat.xsl;
+        }
+
+        location /stat.xsl {
+            root /etc/nginx/;
+        }
+    }
+}