From aa319c856f32543e48a9fb273cde04f8c7ca47cb Mon Sep 17 00:00:00 2001 From: Joey Parrish Date: Fri, 30 Aug 2019 10:57:10 -0700 Subject: [PATCH] Shaka Streamer v0.1.0 This is the first public release of Shaka Streamer! :tada: This initial release was the work of @vickymin13 and @prestontai. Many thanks to both of them for their hard work and dedication! It has been wonderful having them on the team. --- .gitignore | 6 + CODE_OF_CONDUCT.md | 93 ++++ CONTRIBUTING.md | 28 ++ HARDWARE_ENCODING.md | 32 ++ LICENSE | 202 ++++++++ PREREQS.md | 112 +++++ README.md | 53 ++ config_files/input_looped_file_config.yaml | 41 ++ config_files/input_raw_images_config.yaml | 29 ++ config_files/input_vod_config.yaml | 67 +++ config_files/input_webcam_config.yaml | 28 ++ config_files/pipeline_live_config.yaml | 42 ++ .../pipeline_live_encrypted_config.yaml | 57 +++ .../pipeline_live_hardware_config.yaml | 44 ++ config_files/pipeline_vod_config.yaml | 51 ++ .../pipeline_vod_encrypted_config.yaml | 64 +++ package.json | 29 ++ run_end_to_end_tests.py | 239 +++++++++ shaka-streamer-logo.png | Bin 0 -> 7801 bytes shaka_streamer.py | 104 ++++ streamer/__init__.py | 1 + streamer/cloud_node.py | 160 ++++++ streamer/controller_node.py | 263 ++++++++++ streamer/default_config.py | 115 +++++ streamer/discard_node.py | 34 ++ streamer/input_configuration.py | 127 +++++ streamer/loop_input_node.py | 68 +++ streamer/metadata.py | 81 +++ streamer/node_base.py | 76 +++ streamer/packager_node.py | 236 +++++++++ streamer/pipeline_configuration.py | 35 ++ streamer/transcoder_node.py | 261 ++++++++++ streamer/validation.py | 120 +++++ tests/karma.conf.js | 32 ++ tests/tests.js | 465 ++++++++++++++++++ 35 files changed, 3395 insertions(+) create mode 100644 .gitignore create mode 100644 CODE_OF_CONDUCT.md create mode 100644 CONTRIBUTING.md create mode 100644 HARDWARE_ENCODING.md create mode 100644 LICENSE create mode 100644 PREREQS.md create mode 100644 README.md create mode 100644 config_files/input_looped_file_config.yaml create mode 100644 config_files/input_raw_images_config.yaml create mode 100644 config_files/input_vod_config.yaml create mode 100644 config_files/input_webcam_config.yaml create mode 100644 config_files/pipeline_live_config.yaml create mode 100644 config_files/pipeline_live_encrypted_config.yaml create mode 100644 config_files/pipeline_live_hardware_config.yaml create mode 100644 config_files/pipeline_vod_config.yaml create mode 100644 config_files/pipeline_vod_encrypted_config.yaml create mode 100644 package.json create mode 100755 run_end_to_end_tests.py create mode 100644 shaka-streamer-logo.png create mode 100755 shaka_streamer.py create mode 100644 streamer/__init__.py create mode 100644 streamer/cloud_node.py create mode 100644 streamer/controller_node.py create mode 100644 streamer/default_config.py create mode 100644 streamer/discard_node.py create mode 100644 streamer/input_configuration.py create mode 100644 streamer/loop_input_node.py create mode 100644 streamer/metadata.py create mode 100644 streamer/node_base.py create mode 100644 streamer/packager_node.py create mode 100644 streamer/pipeline_configuration.py create mode 100644 streamer/transcoder_node.py create mode 100644 streamer/validation.py create mode 100644 tests/karma.conf.js create mode 100644 tests/tests.js diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..062b076 --- /dev/null +++ b/.gitignore @@ -0,0 +1,6 @@ +*.pyc +output_files/ +test_assets/ +node_modules/ +package-lock.json +Sintel.* diff --git a/CODE_OF_CONDUCT.md b/CODE_OF_CONDUCT.md new file mode 100644 index 0000000..f0cea0e --- /dev/null +++ b/CODE_OF_CONDUCT.md @@ -0,0 +1,93 @@ +# Code of Conduct + +## Our Pledge + +In the interest of fostering an open and welcoming environment, we as +contributors and maintainers pledge to making participation in our project and +our community a harassment-free experience for everyone, regardless of age, body +size, disability, ethnicity, gender identity and expression, level of +experience, education, socio-economic status, nationality, personal appearance, +race, religion, or sexual identity and orientation. + +## Our Standards + +Examples of behavior that contributes to creating a positive environment +include: + +* Using welcoming and inclusive language +* Being respectful of differing viewpoints and experiences +* Gracefully accepting constructive criticism +* Focusing on what is best for the community +* Showing empathy towards other community members + +Examples of unacceptable behavior by participants include: + +* The use of sexualized language or imagery and unwelcome sexual attention or + advances +* Trolling, insulting/derogatory comments, and personal or political attacks +* Public or private harassment +* Publishing others' private information, such as a physical or electronic + address, without explicit permission +* Other conduct which could reasonably be considered inappropriate in a + professional setting + +## Our Responsibilities + +Project maintainers are responsible for clarifying the standards of acceptable +behavior and are expected to take appropriate and fair corrective action in +response to any instances of unacceptable behavior. + +Project maintainers have the right and responsibility to remove, edit, or reject +comments, commits, code, wiki edits, issues, and other contributions that are +not aligned to this Code of Conduct, or to ban temporarily or permanently any +contributor for other behaviors that they deem inappropriate, threatening, +offensive, or harmful. + +## Scope + +This Code of Conduct applies both within project spaces and in public spaces +when an individual is representing the project or its community. Examples of +representing a project or community include using an official project e-mail +address, posting via an official social media account, or acting as an appointed +representative at an online or offline event. Representation of a project may be +further defined and clarified by project maintainers. + +This Code of Conduct also applies outside the project spaces when the Project +Steward has a reasonable belief that an individual's behavior may have a +negative impact on the project or its community. + +## Conflict Resolution + +We do not believe that all conflict is bad; healthy debate and disagreement +often yield positive results. However, it is never okay to be disrespectful or +to engage in behavior that violates the project’s code of conduct. + +If you see someone violating the code of conduct, you are encouraged to address +the behavior directly with those involved. Many issues can be resolved quickly +and easily, and this gives people more control over the outcome of their +dispute. If you are unable to resolve the matter for any reason, or if the +behavior is threatening or harassing, report it. We are dedicated to providing +an environment where participants feel welcome and safe. + +Reports should be directed to *shaka-player-issues@google.com*, the +Project Steward(s) for *Shaka Streamer*. It is the Project Steward’s duty to +receive and address reported violations of the code of conduct. They will then +work with a committee consisting of representatives from the Open Source +Programs Office and the Google Open Source Strategy team. If for any reason you +are uncomfortable reaching out the Project Steward, please email +opensource@google.com. + +We will investigate every complaint, but you may not receive a direct response. +We will use our discretion in determining when and how to follow up on reported +incidents, which may range from not taking action to permanent expulsion from +the project and project-sponsored spaces. We will notify the accused of the +report and provide them an opportunity to discuss it before any action is taken. +The identity of the reporter will be omitted from the details of the report +supplied to the accused. In potentially harmful situations, such as ongoing +harassment or threats to anyone's safety, we may take action without notice. + +## Attribution + +This Code of Conduct is adapted from the Contributor Covenant, version 1.4, +available at +https://www.contributor-covenant.org/version/1/4/code-of-conduct.html diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md new file mode 100644 index 0000000..939e534 --- /dev/null +++ b/CONTRIBUTING.md @@ -0,0 +1,28 @@ +# How to Contribute + +We'd love to accept your patches and contributions to this project. There are +just a few small guidelines you need to follow. + +## Contributor License Agreement + +Contributions to this project must be accompanied by a Contributor License +Agreement. You (or your employer) retain the copyright to your contribution; +this simply gives us permission to use and redistribute your contributions as +part of the project. Head over to to see +your current agreements on file or to sign a new one. + +You generally only need to submit a CLA once, so if you've already submitted one +(even if it was for a different project), you probably don't need to do it +again. + +## Code reviews + +All submissions, including submissions by project members, require review. We +use GitHub pull requests for this purpose. Consult +[GitHub Help](https://help.github.com/articles/about-pull-requests/) for more +information on using pull requests. + +## Community Guidelines + +This project follows [Google's Open Source Community +Guidelines](https://opensource.google.com/conduct/). diff --git a/HARDWARE_ENCODING.md b/HARDWARE_ENCODING.md new file mode 100644 index 0000000..312d115 --- /dev/null +++ b/HARDWARE_ENCODING.md @@ -0,0 +1,32 @@ +# Hardware Encoding with Shaka Streamer + +## Setup on Linux + +Hardware encoding on Linux can be enabled through FFmpeg's vaapi support. + +To get started, install the appropriate vaapi package for your device. For +example, for Intel's Kaby Lake family of processors, which support hardware VP9 +encoding, you would install this on Ubuntu: + +```sh +sudo apt install i965-va-driver +``` + +Or build & install from source here: https://github.com/intel/intel-vaapi-driver + +You will need to install the correct vaapi drivers for your device. These are +only examples. + +If hardware encoding still does not work, you may need to recompile FFmpeg from +source. See instructions in [PREREQS.md](PREREQS.md) for details. + +## Setup on Mac & Windows + +Hardware encoding for Mac & Windows is not yet supported, but we are accepting +PRs if you'd like to contribute additional platform support. This doc may be a +useful reference for hardware-related options in FFmpeg: +https://trac.ffmpeg.org/wiki/HWAccelIntro + +## Configuration + +TODO diff --git a/LICENSE b/LICENSE new file mode 100644 index 0000000..d645695 --- /dev/null +++ b/LICENSE @@ -0,0 +1,202 @@ + + Apache License + Version 2.0, January 2004 + http://www.apache.org/licenses/ + + TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION + + 1. Definitions. + + "License" shall mean the terms and conditions for use, reproduction, + and distribution as defined by Sections 1 through 9 of this document. + + "Licensor" shall mean the copyright owner or entity authorized by + the copyright owner that is granting the License. + + "Legal Entity" shall mean the union of the acting entity and all + other entities that control, are controlled by, or are under common + control with that entity. For the purposes of this definition, + "control" means (i) the power, direct or indirect, to cause the + direction or management of such entity, whether by contract or + otherwise, or (ii) ownership of fifty percent (50%) or more of the + outstanding shares, or (iii) beneficial ownership of such entity. + + "You" (or "Your") shall mean an individual or Legal Entity + exercising permissions granted by this License. + + "Source" form shall mean the preferred form for making modifications, + including but not limited to software source code, documentation + source, and configuration files. + + "Object" form shall mean any form resulting from mechanical + transformation or translation of a Source form, including but + not limited to compiled object code, generated documentation, + and conversions to other media types. + + "Work" shall mean the work of authorship, whether in Source or + Object form, made available under the License, as indicated by a + copyright notice that is included in or attached to the work + (an example is provided in the Appendix below). + + "Derivative Works" shall mean any work, whether in Source or Object + form, that is based on (or derived from) the Work and for which the + editorial revisions, annotations, elaborations, or other modifications + represent, as a whole, an original work of authorship. For the purposes + of this License, Derivative Works shall not include works that remain + separable from, or merely link (or bind by name) to the interfaces of, + the Work and Derivative Works thereof. + + "Contribution" shall mean any work of authorship, including + the original version of the Work and any modifications or additions + to that Work or Derivative Works thereof, that is intentionally + submitted to Licensor for inclusion in the Work by the copyright owner + or by an individual or Legal Entity authorized to submit on behalf of + the copyright owner. For the purposes of this definition, "submitted" + means any form of electronic, verbal, or written communication sent + to the Licensor or its representatives, including but not limited to + communication on electronic mailing lists, source code control systems, + and issue tracking systems that are managed by, or on behalf of, the + Licensor for the purpose of discussing and improving the Work, but + excluding communication that is conspicuously marked or otherwise + designated in writing by the copyright owner as "Not a Contribution." + + "Contributor" shall mean Licensor and any individual or Legal Entity + on behalf of whom a Contribution has been received by Licensor and + subsequently incorporated within the Work. + + 2. Grant of Copyright License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + copyright license to reproduce, prepare Derivative Works of, + publicly display, publicly perform, sublicense, and distribute the + Work and such Derivative Works in Source or Object form. + + 3. Grant of Patent License. Subject to the terms and conditions of + this License, each Contributor hereby grants to You a perpetual, + worldwide, non-exclusive, no-charge, royalty-free, irrevocable + (except as stated in this section) patent license to make, have made, + use, offer to sell, sell, import, and otherwise transfer the Work, + where such license applies only to those patent claims licensable + by such Contributor that are necessarily infringed by their + Contribution(s) alone or by combination of their Contribution(s) + with the Work to which such Contribution(s) was submitted. If You + institute patent litigation against any entity (including a + cross-claim or counterclaim in a lawsuit) alleging that the Work + or a Contribution incorporated within the Work constitutes direct + or contributory patent infringement, then any patent licenses + granted to You under this License for that Work shall terminate + as of the date such litigation is filed. + + 4. Redistribution. You may reproduce and distribute copies of the + Work or Derivative Works thereof in any medium, with or without + modifications, and in Source or Object form, provided that You + meet the following conditions: + + (a) You must give any other recipients of the Work or + Derivative Works a copy of this License; and + + (b) You must cause any modified files to carry prominent notices + stating that You changed the files; and + + (c) You must retain, in the Source form of any Derivative Works + that You distribute, all copyright, patent, trademark, and + attribution notices from the Source form of the Work, + excluding those notices that do not pertain to any part of + the Derivative Works; and + + (d) If the Work includes a "NOTICE" text file as part of its + distribution, then any Derivative Works that You distribute must + include a readable copy of the attribution notices contained + within such NOTICE file, excluding those notices that do not + pertain to any part of the Derivative Works, in at least one + of the following places: within a NOTICE text file distributed + as part of the Derivative Works; within the Source form or + documentation, if provided along with the Derivative Works; or, + within a display generated by the Derivative Works, if and + wherever such third-party notices normally appear. The contents + of the NOTICE file are for informational purposes only and + do not modify the License. You may add Your own attribution + notices within Derivative Works that You distribute, alongside + or as an addendum to the NOTICE text from the Work, provided + that such additional attribution notices cannot be construed + as modifying the License. + + You may add Your own copyright statement to Your modifications and + may provide additional or different license terms and conditions + for use, reproduction, or distribution of Your modifications, or + for any such Derivative Works as a whole, provided Your use, + reproduction, and distribution of the Work otherwise complies with + the conditions stated in this License. + + 5. Submission of Contributions. Unless You explicitly state otherwise, + any Contribution intentionally submitted for inclusion in the Work + by You to the Licensor shall be under the terms and conditions of + this License, without any additional terms or conditions. + Notwithstanding the above, nothing herein shall supersede or modify + the terms of any separate license agreement you may have executed + with Licensor regarding such Contributions. + + 6. Trademarks. This License does not grant permission to use the trade + names, trademarks, service marks, or product names of the Licensor, + except as required for reasonable and customary use in describing the + origin of the Work and reproducing the content of the NOTICE file. + + 7. Disclaimer of Warranty. Unless required by applicable law or + agreed to in writing, Licensor provides the Work (and each + Contributor provides its Contributions) on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or + implied, including, without limitation, any warranties or conditions + of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A + PARTICULAR PURPOSE. You are solely responsible for determining the + appropriateness of using or redistributing the Work and assume any + risks associated with Your exercise of permissions under this License. + + 8. Limitation of Liability. In no event and under no legal theory, + whether in tort (including negligence), contract, or otherwise, + unless required by applicable law (such as deliberate and grossly + negligent acts) or agreed to in writing, shall any Contributor be + liable to You for damages, including any direct, indirect, special, + incidental, or consequential damages of any character arising as a + result of this License or out of the use or inability to use the + Work (including but not limited to damages for loss of goodwill, + work stoppage, computer failure or malfunction, or any and all + other commercial damages or losses), even if such Contributor + has been advised of the possibility of such damages. + + 9. Accepting Warranty or Additional Liability. While redistributing + the Work or Derivative Works thereof, You may choose to offer, + and charge a fee for, acceptance of support, warranty, indemnity, + or other liability obligations and/or rights consistent with this + License. However, in accepting such obligations, You may act only + on Your own behalf and on Your sole responsibility, not on behalf + of any other Contributor, and only if You agree to indemnify, + defend, and hold each Contributor harmless for any liability + incurred by, or claims asserted against, such Contributor by reason + of your accepting any such warranty or additional liability. + + END OF TERMS AND CONDITIONS + + APPENDIX: How to apply the Apache License to your work. + + To apply the Apache License to your work, attach the following + boilerplate notice, with the fields enclosed by brackets "[]" + replaced with your own identifying information. (Don't include + the brackets!) The text should be enclosed in the appropriate + comment syntax for the file format. We also recommend that a + file or class name and description of purpose be included on the + same "printed page" as the copyright notice for easier + identification within third-party archives. + + Copyright [yyyy] [name of copyright owner] + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. diff --git a/PREREQS.md b/PREREQS.md new file mode 100644 index 0000000..d51ed43 --- /dev/null +++ b/PREREQS.md @@ -0,0 +1,112 @@ +# Installing Prerequisites for Shaka Streamer + +## Yaml Module + +We use the Python YAML module to parse config files. To install it on Ubuntu: + +```sh +sudo apt -y install python3-yaml +``` + +This can also be installed via pip on any platform: + +```sh +pip3 install --user pyyaml +``` + +## Shaka Packager + +Pre-built Shaka Packager binaries can be downloaded from github here: +https://github.com/google/shaka-packager/releases + +To install a Shaka Packager binary on Linux: + +```sh +sudo install -m 755 ~/Downloads/packager-linux /usr/local/bin/packager +``` + +To build Shaka Packager from source, follow instructions here: +https://google.github.io/shaka-packager/html/build_instructions.html + +## FFmpeg + +If your Linux distribution has FFmpeg v4.1+, you can just install the package. +For example, this will work in Ubuntu 19.04+: + +```sh +sudo apt -y install ffmpeg +``` + +For older versions of Ubuntu or any other Linux distro which does not have a new +enough version of FFmpeg, you can build it from source. For example: + +```sh +sudo apt -y install \ + libx264-dev libvpx-dev libopus-dev libfreetype6-dev \ + libfontconfig1-dev libsdl2-dev yasm + +git clone https://github.com/FFmpeg/FFmpeg ffmpeg +cd ffmpeg +git checkout n4.1.3 +./configure \ + --enable-libx264 --enable-libvpx --enable-libopus --enable-gpl \ + --enable-libfreetype --enable-libfontconfig --enable-vaapi +make +sudo make install +``` + +For Mac, you can either build FFmpeg from source or you can use +[Homebrew](https://brew.sh/) to install it: + +```sh +brew install ffmpeg +``` + +## Google Cloud Storage (optional) + +Shaka Streamer can push content directly to a Google Cloud Storage bucket. To +use this feature, the Google Cloud SDK and the Cloud Storage python module are +required. + +For Ubuntu, for example, you could install the necessary components like this: + +```sh +sudo apt -y install google-cloud-sdk python3-pip python3-setuptools +pip3 install --user google-cloud +pip3 install --user google-cloud-storage +``` + +See https://cloud.google.com/sdk/install for details on installing the Google +Cloud SDK on your platform. + +If you haven't already, you will need to initialize your gcloud environment and +log in through your browser. + +```sh +gcloud init +gcloud auth login +gsutil config +``` + +Follow the instructions given to you by each of these tools. + +## Test Dependencies (optional) + +To run the end-to-end tests, you must install Flask & NPM. In Ubuntu 19.04+: + +```sh +sudo apt -y python3-flask nodejs npm +# Upgrade to a recent npm, which is not packaged: +sudo npm install -g npm +``` + +Flask can also be installed via pip on any platform: + +```sh +pip3 install --user flask +``` + +To install Node.js & NPM on any other platform, you can try one of these: + - https://github.com/nodesource/distributions + - https://nodejs.org/en/download/ + diff --git a/README.md b/README.md new file mode 100644 index 0000000..3be94b8 --- /dev/null +++ b/README.md @@ -0,0 +1,53 @@ +# ![Shaka Streamer](shaka-streamer-logo.png) + +Shaka Streamer offers a simple config-file based approach to preparing streaming +media. It greatly simplifies the process of using FFmpeg and Shaka Packager for +both VOD and live content. + + +## Platform support + +We support common Linux distributions and macOS. + +Windows is not supported at this time due to our use of `os.mkfifo`, but we are +accepting PRs if you'd like to add Windows support. + + +## Getting started + +Shaka Streamer requires at a minimum: + - [Python 3](https://www.python.org/downloads/) + - [Python "yaml" module](https://pyyaml.org/) + - [Shaka Packager](https://github.com/google/shaka-packager) + - [FFmpeg](https://ffmpeg.org/) + +See the file [PREREQS.md](PREREQS.md) for detailed instructions on installing +prerequisites and optional dependencies. + +To use Shaka Streamer, you need two YAML config files: one to describe the +input, and one to describe the encoding pipeline. Sample configs can be found +in the `config_files/` folder. + +### Example command-line for live streaming to Google Cloud Storage: + +```sh +./main.py \ + -i config_files/input_looped_file_config.yaml \ + -p config_files/pipeline_live_config.yaml \ + -c my_cloud_bucket +``` + +## Running tests + +We have end-to-end tests that will start streams and check them from a headless +browser using Shaka Player. End-to-end tests can be run like so: + +```sh +python3 run_end_to_end_tests.py +``` + +## Hardware encoding + +For details on hardware encoding support, see the file +[HARDWARE_ENCODING.md](HARDWARE_ENCODING.md). + diff --git a/config_files/input_looped_file_config.yaml b/config_files/input_looped_file_config.yaml new file mode 100644 index 0000000..2dbba6a --- /dev/null +++ b/config_files/input_looped_file_config.yaml @@ -0,0 +1,41 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample input configuration file for Shaka Streamer for a looped +# file input. + +# List of inputs. +inputs: + # The type of input. + - input_type: looped_file + # Name of the input file. + # This example can be downloaded from https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.720p.mkv + name: Sintel.2010.720p.mkv + # The track number. + track_num: 0 + # The media type of the input. Can be audio or video. + media_type: video + # Frame rate in seconds. + frame_rate: 24.0 + # Resolution of the input. + resolution: 720p + # Whether or not the video frames are interlaced. + is_interlaced: False + + # The type of input. + - input_type: looped_file + # A second track (audio) from the same input file. + name: Sintel.2010.720p.mkv + track_num: 1 + media_type: audio diff --git a/config_files/input_raw_images_config.yaml b/config_files/input_raw_images_config.yaml new file mode 100644 index 0000000..fc933e4 --- /dev/null +++ b/config_files/input_raw_images_config.yaml @@ -0,0 +1,29 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample input configuration file for Shaka Streamer for raw images as +# input. + +# List of inputs. +inputs: + # The type of input. + - input_type: raw_images + # Name of the input file/pipe. + name: /tmp/ppm_pipe + # The media type of the input. Can be audio or video. + media_type: video + # Frame rate in seconds. + frame_rate: 24.0 + # Resolution of the input. + resolution: 720p diff --git a/config_files/input_vod_config.yaml b/config_files/input_vod_config.yaml new file mode 100644 index 0000000..c6747d5 --- /dev/null +++ b/config_files/input_vod_config.yaml @@ -0,0 +1,67 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample input configuration file for Shaka Streamer for VOD. + +# List of inputs. +inputs: + # Name of the input file. + # This example can be downloaded from https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.4k.mkv + - name: Sintel.2010.4k.mkv + # The track number. + track_num: 0 + # The media type of the input. Can be audio or video. + media_type: video + # Frame rate in seconds. + frame_rate: 24.0 + # Resolution of the input. + resolution: 4k + # Whether or not the video frames are interlaced. + is_interlaced: False + + # A second track (audio) from the same input file. + - name: Sintel.2010.4k.mkv + track_num: 1 + media_type: audio + + # Several text tracks of different languages. + # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Arabic.vtt + - name: Sintel.2010.Arabic.vtt + media_type: text + language: ar + + # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.English.vtt + - name: Sintel.2010.English.vtt + media_type: text + language: en + + # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Esperanto.vtt + - name: Sintel.2010.Esperanto.vtt + media_type: text + language: eo + + # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Spanish.vtt + - name: Sintel.2010.Spanish.vtt + media_type: text + language: es + + # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.French.vtt + - name: Sintel.2010.French.vtt + media_type: text + language: fr + + # https://storage.googleapis.com/shaka-streamer-assets/sample-inputs/Sintel.2010.Chinese.vtt + - name: Sintel.2010.Chinese.vtt + media_type: text + language: zh \ No newline at end of file diff --git a/config_files/input_webcam_config.yaml b/config_files/input_webcam_config.yaml new file mode 100644 index 0000000..71591bc --- /dev/null +++ b/config_files/input_webcam_config.yaml @@ -0,0 +1,28 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample input configuration file for Shaka Streamer for webcam input. + +# List of inputs. +inputs: + # The type of input. + - input_type: webcam + # Name of the input device. + name: /dev/video0 + # The media type. + media_type: video + # Frame rate in seconds. + frame_rate: 24.0 + # Resolution of the input. + resolution: 1080p diff --git a/config_files/pipeline_live_config.yaml b/config_files/pipeline_live_config.yaml new file mode 100644 index 0000000..f27f901 --- /dev/null +++ b/config_files/pipeline_live_config.yaml @@ -0,0 +1,42 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample pipeline configuration file for Shaka Streamer in live mode. +# Here you configure resolutions, manifest formats, segment size, and more. + +# Streaming mode. Can be live or vod. +streaming_mode: live + +transcoder: + # A list of resolutions to encode. + resolutions: + - 720p + - 480p + + # The number of audio channels to output. + channels: 2 + + # The codecs to encode with. + audio_codecs: + - aac + video_codecs: + - h264 + +packager: + # Manifest format (dash, hls, or both) + manifest_format: + - dash + - hls + # Length of each segment in seconds. + segment_size: 4 diff --git a/config_files/pipeline_live_encrypted_config.yaml b/config_files/pipeline_live_encrypted_config.yaml new file mode 100644 index 0000000..2b255f3 --- /dev/null +++ b/config_files/pipeline_live_encrypted_config.yaml @@ -0,0 +1,57 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample pipeline configuration file for Shaka Streamer in live mode. +# Here you configure resolutions, manifest formats, segment size, and more. + +# Streaming mode. Can be live or vod. +streaming_mode: live + +transcoder: + # A list of resolutions to encode. + resolutions: + - 720p + - 480p + + # The number of audio channels to output. + channels: 2 + +packager: + # Manifest format (dash, hls, or both) + manifest_format: + - dash + - hls + # Length of each segment in seconds. + segment_size: 4 + + encryption: + # Enables encryption. + # If disabled, the following settings are ignored. + enable: True + # Content identifier that identifies which encryption key to use. + # This will default to a random content ID, so this is optional. + content_id: '1234' + # Key server url. An encryption key is generated from this server. + key_server_url: https://license.uat.widevine.com/cenc/getcontentkey/widevine_test + # The name of the signer. + signer: widevine_test + # AES signing key in hex string. + signing_key: 1ae8ccd0e7985cc0b6203a55855a1034afc252980e970ca90e5202689f947ab9 + # AES signing iv in hex string. + signing_iv: d58ce954203b7c9a9a9d467f59839249 + # Protection scheme (cenc or cbcs) + # These are different methods of using a block cipher to encrypt media. + protection_scheme: cenc + # Seconds of unencrypted media at the beginning of the stream. + clear_lead: 10 \ No newline at end of file diff --git a/config_files/pipeline_live_hardware_config.yaml b/config_files/pipeline_live_hardware_config.yaml new file mode 100644 index 0000000..e9afac6 --- /dev/null +++ b/config_files/pipeline_live_hardware_config.yaml @@ -0,0 +1,44 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample pipeline configuration file for Shaka Streamer in live mode. +# Here you configure resolutions, manifest formats, segment size, and more. + +# Streaming mode. Can be live or vod. +streaming_mode: live + +transcoder: + # A list of resolutions to encode. + resolutions: + - 720p + - 480p + + # The number of audio channels to output. + channels: 2 + + # The codecs to encode with. + audio_codecs: + - aac + - opus + video_codecs: + - h264 + - hw:vp9 + +packager: + # Manifest format (dash, hls, or both) + manifest_format: + - dash + - hls + # Length of each segment in seconds. + segment_size: 4 diff --git a/config_files/pipeline_vod_config.yaml b/config_files/pipeline_vod_config.yaml new file mode 100644 index 0000000..b885993 --- /dev/null +++ b/config_files/pipeline_vod_config.yaml @@ -0,0 +1,51 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample pipeline configuration file for Shaka Streamer in VOD mode. +# Here you configure resolutions, manifest formats, segment size, and more. + +# Streaming mode. Can be live or vod. +streaming_mode: vod + +transcoder: + # A list of resolutions to encode. + # For VOD, you can specify many more resolutions than you would with live, + # since the encoding does not need to be done in real time. + resolutions: + - 4k + - 1080p + - 720p + - 480p + - 360p + + # The number of audio channels to output. + channels: 6 + + # The codecs to encode with. + audio_codecs: + - aac + - opus + video_codecs: + - h264 + - vp9 + +packager: + # Manifest format (dash, hls or both) + manifest_format: + - dash + - hls + # Length of each segment in seconds. + segment_size: 10 + # Forces the use of SegmentTemplate in DASH. + segment_per_file: True diff --git a/config_files/pipeline_vod_encrypted_config.yaml b/config_files/pipeline_vod_encrypted_config.yaml new file mode 100644 index 0000000..dad6ea5 --- /dev/null +++ b/config_files/pipeline_vod_encrypted_config.yaml @@ -0,0 +1,64 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# This is a sample pipeline configuration file for Shaka Streamer in VOD mode. +# Here you configure resolutions, manifest formats, segment size, and more. + +# Streaming mode. Can be live or vod. +streaming_mode: vod + +transcoder: + # A list of resolutions to encode. + # For VOD, you can specify many more resolutions than you would with live, + # since the encoding does not need to be done in real time. + resolutions: + - 4k + - 1080p + - 720p + - 480p + - 360p + + # The number of audio channels to output. + channels: 6 + +packager: + # Manifest format (dash, hls or both) + manifest_format: + - dash + - hls + # Length of each segment in seconds. + segment_size: 10 + # Forces the use of SegmentTemplate in DASH. + segment_per_file: True + + encryption: + # Enables encryption. + # If disabled, the following settings are ignored. + enable: True + # Content identifier that identifies which encryption key to use. + # This will default to a random content ID, so this is optional. + content_id: '1234' + # Key server url. An encryption key is generated from this server. + key_server_url: https://license.uat.widevine.com/cenc/getcontentkey/widevine_test + # The name of the signer. + signer: widevine_test + # AES signing key in hex string. + signing_key: 1ae8ccd0e7985cc0b6203a55855a1034afc252980e970ca90e5202689f947ab9 + # AES signing iv in hex string. + signing_iv: d58ce954203b7c9a9a9d467f59839249 + # Protection scheme (cenc or cbcs) + # These are different methods of using a block cipher to encrypt media. + protection_scheme: cenc + # Seconds of unencrypted media at the beginning of the stream. + clear_lead: 10 \ No newline at end of file diff --git a/package.json b/package.json new file mode 100644 index 0000000..bd39e9d --- /dev/null +++ b/package.json @@ -0,0 +1,29 @@ +{ + "name": "shaka-streamer", + "description": "Simple tool for live video streaming and VOD content preparation.", + "version": "1.0.0", + "homepage": "https://github.com/google/shaka-streamer", + "author": "Google", + "maintainers": [ + { + "name": "Joey Parrish", + "email": "joeyparrish@google.com" + } + ], + "devDependencies": { + "karma": "^4.1.0", + "karma-chrome-launcher": "^3.0.0", + "karma-jasmine": "^2.0.1", + "karma-junit-reporter": "^1.2.0", + "karma-spec-reporter": "^0.0.32", + "shaka-player": "^2.5.5" + }, + "repository": { + "type": "git", + "url": "https://github.com/google/shaka-streamer.git" + }, + "bugs": { + "url": "https://github.com/google/shaka-streamer/issues" + }, + "license": "Apache-2.0" +} diff --git a/run_end_to_end_tests.py b/run_end_to_end_tests.py new file mode 100755 index 0000000..71326e2 --- /dev/null +++ b/run_end_to_end_tests.py @@ -0,0 +1,239 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +import argparse +import flask +import glob +import json +import os +import re +import shutil +import subprocess +import sys +import time +import threading +import urllib + +from streamer.controller_node import ControllerNode + +OUTPUT_DIR = 'output_files/' +TEST_DIR = 'test_assets/' +CLOUD_TEST_ASSETS = ( + 'https://storage.googleapis.com/shaka-streamer-assets/test-assets/') + +# Changes relative path to where this file is. +os.chdir(os.path.dirname(__file__)) +controller = None + +app = flask.Flask(__name__, static_folder=OUTPUT_DIR) +# Stops browser from caching files to prevent cross-test contamination. +app.config['SEND_FILE_MAX_AGE_DEFAULT'] = 0 + +def cleanupFiles(): + # Check if the directory for outputted Packager files exists, and if it + # does, delete it and remake a new one. + if os.path.exists(OUTPUT_DIR): + shutil.rmtree(OUTPUT_DIR) + os.mkdir(OUTPUT_DIR) + +def hasSegment(representation): + return re.search('') + while missing_segment: + time.sleep(1) + with open(dash_path) as dash_file: + missing_segment = False + for representation in pattern.finditer(dash_file.read()): + if not hasSegment(representation): + missing_segment = True + +def hlsReadyStreamCount(stream_list): + init_count = 0 + for stream_path in stream_list: + with open(stream_path) as stream_file: + if '#EXTINF' in stream_file.read(): + init_count += 1 + return init_count + +def waitHlsManifest(hls_path): + # Does not read manifest until it is created. + while not os.path.exists(hls_path): + time.sleep(1) + + # Parsing master playlist to see how many streams there are. + stream_pattern = re.compile('stream_\d+\.m3u8') + with open(hls_path) as hls_file: + stream_count = len(set(stream_pattern.findall(hls_file.read()))) + + # Waiting until the correct number of streams exist. + stream_path_glob = OUTPUT_DIR + 'stream_*.m3u8' + while len(glob.glob(stream_path_glob)) != stream_count: + time.sleep(1) + + # Waiting until each stream has enough segments. + stream_list = glob.glob(stream_path_glob) + while hlsReadyStreamCount(stream_list) != stream_count: + time.sleep(1) + +@app.route('/start', methods = ['POST']) +def start(): + global controller + if controller is not None: + return createCrossOriginResponse( + status=403, body='Instance already running!') + cleanupFiles() + + # Receives configs from the tests to start Shaka Streamer. + configs = json.loads(flask.request.data) + + controller = ControllerNode() + try: + controller.start(OUTPUT_DIR, configs['input_config'], + configs['pipeline_config']) + except: + # If the controller throws an exception during startup, we want to call + # stop() to shut down any external processes that have already been started. + # Then, re-raise the exception. + controller.stop() + raise + + return createCrossOriginResponse() + +@app.route('/stop') +def stop(): + global controller + if controller is not None: + controller.stop() + controller = None + cleanupFiles() + return createCrossOriginResponse() + +@app.route('/output_files/', methods = ['GET','OPTIONS']) +def send_file(filename): + if controller.is_vod(): + # If streaming mode is vod, needs to wait until packager is completely + # done packaging contents. + while controller.is_running(): + time.sleep(1) + else: + # If streaming mode is live, needs to wait for specific content in + # manifest until it can be loaded by the player. + if filename == 'output.mpd': + waitDashManifest(OUTPUT_DIR + 'output.mpd') + elif filename == 'master_playlist.m3u8': + waitHlsManifest(OUTPUT_DIR + 'master_playlist.m3u8') + + # Sending over requested files. + try: + response = flask.send_file(OUTPUT_DIR + filename); + except FileNotFoundError: + response = flask.Response(response='File not found', status=404) + + response.headers.add('Access-Control-Allow-Origin', '*') + response.headers.add('Access-Control-Allow-Headers', 'RANGE') + return response + +def fetch_cloud_assets(): + file_list = [ + 'BigBuckBunny.1080p.mp4', + 'Sintel.2010.720p.Small.mkv', + 'Sintel.2010.Arabic.vtt', + 'Sintel.2010.Chinese.vtt', + 'Sintel.2010.English.vtt', + 'Sintel.2010.Esperanto.vtt', + 'Sintel.2010.French.vtt', + 'Sintel.2010.Spanish.vtt', + ] + + # Downloading all the assests for tests. + for file in file_list: + if not os.path.exists(TEST_DIR + file): + response = urllib.request.urlretrieve(CLOUD_TEST_ASSETS + + file, + TEST_DIR + file) + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--runs', default=1, type=int, + help='Number of trials to run') + parser.add_argument('--reporters', nargs='+', + help='Enables specified reporters in karma') + args = parser.parse_args() + + # Install test dependencies. + subprocess.check_call(['npm', 'install']) + + # Fetch streams used in tests. + if not os.path.exists(TEST_DIR): + os.mkdir(TEST_DIR) + + fetch_cloud_assets() + + # Start up flask server on a thread. + # Daemon is set to True so that this thread automatically gets + # killed when exiting main. Flask does not have any clean alternatives + # to be killed. + threading.Thread(target=app.run, daemon=True).start() + + fails = 0 + trials = args.runs + print('Running', trials, 'trials') + # Start up karma. + for i in range(trials): + # Start up karma. + karma_args = [ + 'node_modules/karma/bin/karma', + 'start', + 'tests/karma.conf.js', + # DRM currently is not compatible with headless, so it's run in Chrome. + # Linux: If you want to run tests as "headless", wrap it with "xvfb-run -a". + '--browsers', 'Chrome', + '--single-run', + ] + + if args.reporters: + converted_string = ','.join(args.reporters) + karma_args += [ + '--reporters', + converted_string, + ] + # If the exit code was not 0, the tests in karma failed or crashed. + if subprocess.call(karma_args) != 0: + fails += 1 + + print('\n\nNumber of failures:', fails, '\nNumber of trials:', trials) + print('\nSuccess rate:', 100 * (trials - fails) / trials, '%') + return fails + +if __name__ == '__main__': + # Exit code based on test results from subprocess call. + sys.exit(main()) diff --git a/shaka-streamer-logo.png b/shaka-streamer-logo.png new file mode 100644 index 0000000000000000000000000000000000000000..48dc2ab1b55d37a629d3d136da6e9929e9eea894 GIT binary patch literal 7801 zcmZu$by$=^x2Kk+yQDi8B&9(b1?kwOT)JCe0RaK&Py`lGN_vr4K)QC3kX9O`73q== z;jZ6zpZnMSW8U|fb7tN-Gjr;9-UI_ZEm9&zA}lN{(kI&LMws_kOq(OX!8|$k6ZEphTiHzeeV zp7U;r#1n08cH-^FJr162=~x^fe3JhDl=&Vv)EomYel0`oTz`^BFq*g@o^lbE<~aYZ zYILDhHY~iQnW`#ec^xZUzpq`3e5aj8IfXcBF(fo zo2+4DH5(Q~MH$JOsG%(cwC6p4=eO4=9Mf428@2~VT_vLJG;ww}Cf+zDGZN8tf7T4$ zjJJ7I5j`PRVmxM{+JrD$%ILiFAJ=Y$w~n2b#%#cTr@mLy_Rvt4>1@$P`%%8o(plMC z8~I3jyj1na(a5{)g^`jy86r?*i;vlKoUlAmpsbB0$?Wf66+$Yoft%KQ0kFP4XzucYA$Zo%JF8cusCU1G#|4QMYS|du`m85&a;#t+i0N%VLV_&Y+Jbc)zQ8 zSbDAQ%wV~z5?y&Xaru%Y`m|}CU09YG z^Za2u(aPm1*E75TD^})OvVR6Tr09Q{ZWahzM_D)ti4l0nR`F`}byHvun_QiX40eb);p4R6to8V3IQdIN1~#btVOr!{ z$CTVZ6KHLd#&MGR+pJ8MPwQcILvpe&lk!Sd0yf<6dQkeEC(Oe!h&~bg`5B0XFEyh< z@@ym)2xfYCRuRSq-Ne^&*3X+(IwA2k?7sQ43PDKb zwMVQQn%NQG3vasqbTQo3XaA|NxyQf5Vxi`G%TFI#5%tcv{4ur;r!i&iu5&`CO>_rb zHfh1v%)U%_w+1jy`}=Pincpt;6)RekI_%tLcvf;7HCN>08rJ}#}hF_q?j>- zUHzj%*SyxHGF$uv%_lcVsnC>R8u9%39WAS64dS@i*dI6njMku~0~Z$Y@@NLwYX0;h zHS}j6(nl8R?yq>(UQ@GLPE^V$AZ-MfWbCKF`^3TIWl`}oB4EZ&Qbhmk5FgC!nwRDd zV_DmlamOe>6U87L31&9KP``ncyF%j_0=g};OlHJH9@id;wLL^u06BFSE@Rn7`bIc- zx`~vwV7>0_cXhi;xuA$M10^WCw=>T8(A(&GWTyO&q?3L=2Y`-dI;}HsknLzrusXFL zcQwu&4%uj+)l|fV2X?C+QE95rOR|%djFHgFt(FERWs#EBw4@>0XY<onN5dIIjc6wS+xgy&UvV7wMd=3xZid;;Fg?gK<* zN#(CK$?I6tl#Wcw+-Q4S*}o?L<_I-$dnX(4 z9a{mJJfVNjaYe>K)UAYN5|%%;JMJVX4t+41I-YLeS0in+v#ppc{Jj^OK!o}~&C zjAb6*bK|{pe#@JY%pujdsw&F~e%MteG#@ztFM^%uTr^SP?Phj>6)L$sCDfZEr9LomB5ju>O&{6G; z1FgZp(c+HBj~8lG89$3yxleZdkz=(qM#LkI`)_J*rG&oMese)987by~<;ax8mVH6| zP~n2eP@ucK<3579_rbb6TVYxMW~pNrJPhI>ZZ_z|IY0QE`)mk5K<8^u$Ohra!?&pG z)2FpBcWo4!42*yB9eT#_cBH`tI5)$V8r|+g51mREUC4S}^WqOD&^jslEel2A=RCc> zLK0No->#KD0^TD_-WfgH&Aj!4i6;?ym88N*j0oM&_TFFn$=iKBRnGwM zNp%TIu7VCL8M77};2^U^Cpm-dw298V$$@*Sh*8~9 z#I~ujzJ@bD1umgb+$a+B3I`89FL%!}Hg7Rpp3uw95!={xH)L{*t%5Z24U~w0&sokQ zYzT7boA3|vzAd>Rj>VcorY*bFnLn>TiEV7+`Al^fDFNWR>oyaxLgQF#>7;m%d8+HC z+%Glk^!#}qJb344FgVOw{Qx1hh$ zUHaf^rR_FWU~|xx2GraG!_~d-{WDqh$y-wg_m-k20s-4=&|$fd@OnRVx0P90*TWK* z&UouF8+oeo+p1jD=o`w2tE6B0LMiwxNAN(4y9rbYDHi~C{FxL75_5Xc-Vd!0TAi;b z2FLw07EH)(qdq;$@xdYH-Z!9pKH^;CF*!1Jw?RtdE+aZ5wQ)EQl&4#R9h#K|&wNpQ zIX7)##Pk>ETyTLFFyrT@@Lifz7HyH5WBSBUvuegn!T2D=P9vaQkQrF{7Rx>1gLQ!_q@VpfxlWpx;ecA{S#MFen{6sI+(UJcVMnvu6iDZy>CQtXzQNknu4yQc~0DuNbQN*IH@vmd2Cqq3ZC%|YK#hf!<#0_(h5 zCqla&j#Zo zhCQU{rE*)3(cM(r>YktPC!|iH+^QQyodHTAoI+q}qlS9|=dp;GDYHN27&+?3_ z$4Kzo$ZpkNR%&sY#iJuRVSk^I>t%{?4HromshcT*dpD2^ql#hIJ$BREurFy`22S_E zx&@i`F8e;&!Zc+6FP{8|G%*a*_$bFZu$;X7$;nBrRcrkZjm@#V;ju9vkwrk*rT9 zm0ew3Wo2dVvo*)zGKz|MQ5xuLja3d7`M_l2r)p}^@|#Z(CN^C!b8a$IQ!$$Y>?UC3 zhOhwdZlcRXJ)Jv|*G+e|&U zGjOd3P!MG^(u~TEy1^Vkg+K{ZSm)DpeaJJ;w96NORaX}mm+wD`ZTZ2U$Us|MX0?rX zFLfWN7$Udg8{0cNaxubj9@?`ghrbLDS2}{udpz>FZI76_yu4h@?H+)rshJ4UzFq6s zo14qFsU>$rN-~nR(2WsYTwDN??R%iU#rP>HDUo>YZf-kcdGZ}9v;Lc7jg5`L)=3Vo zwzk;3O%Mp=V1{sY4N*~n_4^1FRbKQ#-#~hKT#b*0hDKdW>%{Ex+efw5)>bYqt~ETf z&`|ki9${f&Y%JoC6&9nKD1pu1-q)xVBYqnChc+}VRDhsi7fC80@j}4wkp~CkD#}OU zGV=14(3pqMyFgjPUvNPTC}EuJmmf3HtZC0W5o$^HHPM9))ybqaw79mJoT%N7?E zp_HibhWRQBsbTUf4LiHLU$8(F3;{u&+WPvypg0h?T)!YT^rw6H*m7rR=F`%+=ZKmb zy3Y86NQ(j#PUTp>qEN0N(sy0Z3Ich64~<;qIQo`M&8IgtJssRt+k=h*zHgi&c#DnB zRNkJcwiFW+>zIyz^Cn2T4y$~~dV`vZDlsW(Z73!t#s{!GuCA%s=HL1{+EF|QB|xK{ zsaNiQ;ew6^K4KIkk$4fLbJx3+$#OTF0AkXJ@{PYo~^s(DYSh%snisi&ebDh**2GP7Xd&>zHa|B8GgjO-@WqYT5IO2k>Mz8xdW;NjEe1Co( z;tNHGoym1Z2{2uppY%7Qbh9!tUa|FCLB1A~kmK(TiON%B!rh+^CfC$Gv0}lV4DU8g=C|*|$Z4|3w2zLoDRf780mi4`m zie#V7;>yaz0N)z&%?F^CT50cP8tFL4F0$E&{QMMPTZU<{@~_~hOn%T-CGcY`nW_Ba zC~_DEM-q(*B;#$47hJ5) zMpFw@fP6m1#LeI4x<)tG+(8zxU01ro@^gHEbgZVL2XaCqqv}y>qi36ff`UCvpxN)) zvxWdiM@K?D9T+#ui&Yf%b2JVvHrvmTc3sX0P0fDHRphh^kKn7n(#RET;Z5$vI4Vj? zN-JmFvbR!FQo2+;?h={pO61DO+^VBkTa*Mf4xA!u1yNGMy#_^({OnDj#`%)`m|>WO z4u=l*x5I>omp5_iNsQ{+^rTNp0+Q46XfqLq*g=$`#hqR>n-F`rp<#St`jdIgE5Mm8 zsj|B2krfhV?(y<{noRXeS64&7nw~CQU>soq%VH=r!cYBOQ7h;Nl1LM9=yaM zcB0M8%UevFa3^>rZMuEBJ=wupx!b0V@$NE_eXqA;V`F>6@)~;xxL&fOqk>;;^+mYI z2nFlNfFDn9ZB?`%)oy}R%|MCA&lC$H)@XJ?g%kkdgQupZwCODb{?@+qvLUAxn`K2& z0C-P9+~q1dniJ(`wBhv?&qi^NYcs^D=LDIQn5fE9g|Ry`Vkt(khYrqWTUMS1?C5n1 z;F^l~W|#Iq#{ec4cm4VQA$%Hl#&$ zxn3UC>|m8VExwN28#ZOS&eRVD1&3%I0u-m@EZ2GEyLVIv3P}Xu4t&b%nSj~GcvJr) zmuI}jmyvzh{v5#vReuljo3rrnt9UL1><+3PrME@n(H)Xdl86c69dh8tm1bnnIJL3B zp|Whe2h90FT>LO$Rijy4TsrEzqYyG28HbulNp9rg19h{vAYL}5{C(-^deT!5M^Fo{CpU~ zWXJgF3lyFRJr-%WoPw&Z`QcJKeOntQd#SB`vx;B)7AwiEG2#EJ^8cy5|8K7JpQ`)@ zL@v(@2FsqpzBi$%+E@f0JkS&n5SS}pFs=q8naScdrCgJXKO!(`+11sRjRZ@}hoQVD z4dPM6qM6c>ND;rIPGIcnoc&HVi02?0Jne(vT_ zU7(EtCR57KNK0GO-TnFZv85%`;SC-?mtCKqxR0n_3+ugRj~p=9sP9gqTL&W}F0 zoNfo^^@NX%jm@0DxNxexzau4@!p9|(?W&ezbXm+}xaH$o#BzP~%w1n!zoqzefY>J> zH`~B^ZGd`rb@eg7kzl5UAMt>`6nYq0w#?C*(AX%|+~-xV2wk4tZB)ww_>FFbn+J_vK4wOD0lfaWerCez}6&JTx@Kwmz95e5+AYcv#?W z^+hyn%j(k6)s^rNK2zbD1rDi3OtILSp-pIr01C*LO5?x=&l8iJZ+>-PJnOj;*g|^; zlw)QS6Tc6OSY_PM!zesJij$KQGA4L?J7Xx`75%2r=;)9G{^8AMkrd0Dn4rAv7>;%? z)zqAVg6{A*1`=-yK0ZDbN(GKfPCdj29saS=(YjQnWmvG@C5L$&Vs38EhY7Mp+(54E zac6=G+uGfgmy`1&W4(HLG>qNz3=n#rkCG<#c#ibLM}~oytc-CXdw!R>~$zX zCBO^a%Qh5gA{o)6#0^U;tEr%{fPm7>gB-A^;qxZm%?J)UW!cNIWh0pm=jHekJ~*7! zcDeV3w*x{oh`ra$rd0k~pszwlK=Y;Mh-AqtCCX3&275U6f zw_FEbQc_eL6{&livA^%RS=x`68N9I0|C_5B0ss0PM2jLCn+)DBA(}pOg&;7 zD4W!4=RK89Mr;A%MQg?PJsr8Zxv67_>I@fgeK2CC6I&G8yI%Ov4FCMkpNS;gdhM#& zS@Rj-WHC9LP9ouNKiOrTJ~xW4Mx84B%=h0BJNOAYw%7irc6Qr#N^^ez#%n)*{5U#7 zz9wFn;ZzR@!jpc!m1)Q}JQ=0s0muT>a#cl{b7H4b7JMyA_K7}PJ+3d~oB8`!(VgE; zGw@{Z=T9y<)bkun)L#^M2kx<%2iMMaCcnIh-*~nj|7Uemc3D|L!Dl3%b*TJ!PwvoS z&{{2t2e?bIJrdgEQkFm}`)S+yp9Q51UoeYxTJyj+nR-#wj)4i&E^y&z6vIMvNed9N zr#9vo#MtX531GLMjd_rc5S-wzlMsx+h9yy#de~82to%sYzU51) zkw8Va@Z!6|353z%-d>A!Hgl(g3d#8R_=aB&Jpm?D_4M-UzjHMGG!34rHQ<-2le(0? zPNcd8p4?0f7NR1MDt9of$v##$rGk|oKm4CFO*W@jf&5-N z`ULG2XQ(651VvL0Lq+W#B!ls};OPG&qT?O^OZ>=dHjkF2vvJ`kR44 S0_I;b))Ngq^%_-Z)c*jN-S^x8 literal 0 HcmV?d00001 diff --git a/shaka_streamer.py b/shaka_streamer.py new file mode 100755 index 0000000..1a1f3f7 --- /dev/null +++ b/shaka_streamer.py @@ -0,0 +1,104 @@ +#!/usr/bin/env python3 +# +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +""" +Shaka Streamer {version} + +Shaka Streamer offers a simple config-file based approach to preparing streaming +media. It greatly simplifies the process of using FFmpeg and Shaka Packager for +both VOD and live content. +""" + +import argparse +import os +import shutil +import time +import yaml + +from streamer import VERSION +from streamer.controller_node import ControllerNode + + +class CustomArgParseFormatter( + argparse.ArgumentDefaultsHelpFormatter, + argparse.RawDescriptionHelpFormatter): + """A custom argparse formatter that combines the features of multiple base + classes. This gives us defaults for each argument in the help text, plus + it preserves whitespace in the description field.""" + pass + + +def main(): + description = __doc__.format(version=VERSION) + + parser = argparse.ArgumentParser(description=description, + formatter_class=CustomArgParseFormatter) + + parser.add_argument('-i', '--input_config', + required=True, + help='The path to the input config file (required).') + parser.add_argument('-p', '--pipeline_config', + required=True, + help='The path to the pipeline config file (required).') + parser.add_argument('-c', '--cloud_url', + default=None, + help='The Google Cloud Storage URL to upload to.') + parser.add_argument('-o', '--output', + default='output_files', + help='The output folder to write files to. ' + + 'Used even if uploading to cloud storage.') + + args = parser.parse_args() + + # Check if the directory for outputted Packager files exists, and if it + # does, delete it and remake a new one. + if os.path.exists(args.output): + shutil.rmtree(args.output) + os.mkdir(args.output) + + controller = ControllerNode() + + with open(args.input_config) as f: + input_config_dict = yaml.load(f) + with open(args.pipeline_config) as f: + pipeline_config_dict = yaml.load(f) + + if args.cloud_url: + if not args.cloud_url.startswith('gs://'): + parser.error('Invalid cloud URL, only gs:// URLs are supported currently') + + try: + controller.start(args.output, input_config_dict, pipeline_config_dict, + args.cloud_url) + except: + # If the controller throws an exception during startup, we want to call + # stop() to shut down any external processes that have already been started. + # Then, re-raise the exception. + controller.stop() + raise + + # Sleep so long as the pipeline is still running. + while controller.is_running(): + try: + time.sleep(1) + except KeyboardInterrupt: + # Sometimes ffmpeg/packager take a while to be killed, so this signal + # handler will kill both running processes as there is SIGINT signal. + controller.stop() + break + +if __name__ == '__main__': + main() diff --git a/streamer/__init__.py b/streamer/__init__.py new file mode 100644 index 0000000..07a9a63 --- /dev/null +++ b/streamer/__init__.py @@ -0,0 +1 @@ +VERSION = "v0.1.0" diff --git a/streamer/cloud_node.py b/streamer/cloud_node.py new file mode 100644 index 0000000..4cac301 --- /dev/null +++ b/streamer/cloud_node.py @@ -0,0 +1,160 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Pushes output from packager to cloud.""" + +from google.cloud import storage + +import glob +import json +import os +import shutil +import sys +import threading +import time + +from . import node_base + +# This is the value for the HTTP header "Cache-Control" which will be attached +# to the Cloud Storage blobs uploaded by this tool. When the browser requests +# a file from Cloud Storage, the server will use this as the value of the +# "Cache-Control" header it returns. +# Here "no-store" means that the response must not be stored in a cache, and +# "no-transform" means that the response must not be manipulated in any way +# (including Chrome's data saver features which might want to re-encode +# content). +# https://developer.mozilla.org/en-US/docs/Web/HTTP/Headers/Cache-Control +CACHE_CONTROL_HEADER = 'no-store, no-transform' + +class CloudNode(node_base.NodeBase): + + def __init__(self, input_dir, bucket_url, temp_dir): + node_base.NodeBase.__init__(self) + self._input_dir = input_dir + self._temp_dir = temp_dir + self._storage_client = storage.Client() + self._running = True + self._bucket_url = bucket_url + bucket, path = self._bucket_url.replace('gs://', '').split('/', 1) + self._bucket = self._storage_client.get_bucket(bucket) + # Strip trailing slashes to make sure we don't construct paths later like + # foo//bar, which is _not_ the same as foo/bar in Google Cloud Storage. + self._subdir_path = path.rstrip('/') + self._thread = threading.Thread(target=self._thread_main, name='cloud') + + def _thread_main(self): + while self._running: + try: + self._upload_all() + except: + print('Exception in cloud upload:', sys.exc_info()) + print('Cloud upload continuing.') + + # Yield time to other threads. + time.sleep(1) + + def _upload_all(self): + all_files = os.listdir(self._input_dir) + is_manifest_file = lambda x: x.endswith('.mpd') or x.endswith('.m3u8') + manifest_files = filter(is_manifest_file, all_files) + segment_files = filter(lambda x: not is_manifest_file(x), all_files) + + # The manifest at any moment will reference existing segment files. + # We must be careful not to upload a manifest that references segments that + # haven't been uploaded yet. So first we will capture manifest contents, + # then upload current segments, then upload the manifest contents we + # captured. + + manifest_contents = {} + for filename in manifest_files: + source_path = os.path.join(self._input_dir, filename) + contents = b'' + + # Capture manifest contents, and retry until the file is non-empty or + # until the thread is killed. + while not contents and self._running: + time.sleep(0.1) + + with open(source_path, 'rb') as f: + contents = f.read() + + manifest_contents[filename] = contents + + for filename in segment_files: + # Check if the thread has been interrupted. + if not self._running: + return + + source_path = os.path.join(self._input_dir, filename) + destination_path = self._subdir_path + '/' + filename + self._sync_file(source_path, destination_path) + + for filename, contents in manifest_contents.items(): + # Check if the thread has been interrupted. + if not self._running: + return + + destination_path = self._subdir_path + '/' + filename + self._upload_string(contents, destination_path) + + # Finally, list blobs and delete any that don't exist locally. This will + # help avoid excessive storage costs from content that is outside the + # availability window. We use the prefix parameter to limit ourselves to + # the folder this client is uploading to. + all_blobs = self._storage_client.list_blobs(self._bucket, + prefix=self._subdir_path + '/') + for blob in all_blobs: + # Check if the thread has been interrupted. + if not self._running: + return + + assert blob.name.startswith(self._subdir_path + '/') + filename = blob.name.replace(self._subdir_path + '/', '') + local_path = os.path.join(self._input_dir, filename) + if not os.path.exists(local_path): + blob.delete() + + def _sync_file(self, source_file, dest_blob_name): + blob = self._bucket.blob(dest_blob_name) + blob.cache_control = CACHE_CONTROL_HEADER + + try: + if blob.exists(self._storage_client): + blob.reload(self._storage_client) + modified_datetime = os.path.getmtime(source_file) + if modified_datetime <= blob.updated.timestamp(): + # We already have an up-to-date copy in cloud storage. + return + + blob.upload_from_filename(source_file) + + except FileNotFoundError: + # The file was deleted by the Packager between the time we saw it and now. + # Ignore this one. + return + + def _upload_string(self, source_string, dest_blob_name): + blob = self._bucket.blob(dest_blob_name) + blob.cache_control = CACHE_CONTROL_HEADER + blob.upload_from_string(source_string) + + def start(self): + self._thread.start() + + def stop(self): + self._running = False + self._thread.join() + + def is_running(self): + return self._running diff --git a/streamer/controller_node.py b/streamer/controller_node.py new file mode 100644 index 0000000..762a211 --- /dev/null +++ b/streamer/controller_node.py @@ -0,0 +1,263 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""Controls other modules and shared resources. + +This is the main module, which instantiates and starts other modules, and which +manages shared resources like named pipes.""" + +import os +import re +import shutil +import string +import subprocess +import tempfile +import uuid + +from . import input_configuration +from . import loop_input_node +from . import metadata +from . import packager_node +from . import pipeline_configuration +from . import transcoder_node + +class VersionError(Exception): + """Raised when a version is not new enough to work with Shaka Streamer.""" + pass + +class ControllerNode(object): + + def __init__(self): + global_temp_dir = tempfile.gettempdir() + + # The docs state that if any of prefix, suffix, or dir are specified, all + # must be specified (and not None). Create a temp dir of our own, inside + # the global temp dir, and with a name that indicates who made it. + self._temp_dir = tempfile.mkdtemp( + dir=global_temp_dir, prefix='shaka-live-', suffix='') + + self._nodes = [] + + def __del__(self): + # Clean up named pipes by removing the temp directory we placed them in. + shutil.rmtree(self._temp_dir) + + def _create_pipe(self): + """Create a uniquely-named named pipe in the node's temp directory. + + Raises: + RuntimeError: If the platform doesn't have mkfifo. + Returns: + The path to the named pipe, as a string. + """ + + # TODO: mkfifo only works on Unix. We would need a special case for a + # Windows port some day. + + if not hasattr(os, 'mkfifo'): + raise RuntimeError('Platform not supported due to lack of mkfifo') + + # Since the tempfile module creates actual files, use uuid to generate a + # filename, then call mkfifo to create the named pipe. + unique_name = str(uuid.uuid4()) + path = os.path.join(self._temp_dir, unique_name) + + readable_by_owner_only = 0o600 # Unix permission bits + os.mkfifo(path, mode=readable_by_owner_only) + + return path + + def start(self, output_dir, input_config_dict, pipeline_config_dict, bucket_url=None): + """Create and start all other nodes.""" + if self._nodes: + raise RuntimeError('Controller already started!') + + # Check that ffmpeg version is 4.1 or above. + check_version('FFmpeg', ['ffmpeg', '-version'], (4, 1)) + + # Check that Shaka Packager version is 2.1 or above. + check_version('Packager', ['packager', '-version'], (2, 1)) + + input_config = input_configuration.InputConfig(input_config_dict) + + pipeline_config = pipeline_configuration.PipelineConfig( + pipeline_config_dict) + self.pipeline_config = pipeline_config + + # Some inputs get processed by Shaka Streamer before being transcoded, so + # this array will keep track of the inputs to pass to the transcoder. Some + # will be input files/devices, while others will be named pipes. + processed_inputs = [] + + # By default, assume there is no audio or video input. In the loop, + # these variables can only be set to True, to detect if there is at least + # one audio and video stream. + has_audio_input = False + has_video_input = False + has_text_input = False + + for i in input_config.inputs: + # Check (based on information from the configuration) if the input + # file has audio or video. + if not has_audio_input: + has_audio_input = i.has_audio() + if not has_video_input: + has_video_input = i.has_video() + if not has_text_input: + has_text_input = i.has_text() + + if pipeline_config.mode == 'live': + i.check_live_validity() + if i.get_input_type() == 'looped_file': + loop_output = self._create_pipe() + input_node = loop_input_node.LoopInputNode(i.get_name(), loop_output) + self._nodes.append(input_node) + processed_inputs.append(loop_output) + + elif i.get_input_type() == 'raw_images': + processed_inputs.append(i.get_name()) + + elif i.get_input_type() == 'webcam': + processed_inputs.append(i.get_name()) + + elif pipeline_config.mode == 'vod': + i.check_vod_validity() + processed_inputs.append(i.get_name()) + + audio_outputs = [] + if has_audio_input: + for i in input_config.inputs: + audio_outputs.extend(self._add_audio(i, pipeline_config.transcoder['channels'], + pipeline_config.transcoder['audio_codecs'])) + + video_outputs = [] + if has_video_input: + for i in input_config.inputs: + video_outputs.extend(self._add_video(i, + pipeline_config.transcoder['resolutions'], + pipeline_config.transcoder['video_codecs'])) + + # Process input through a transcoder node using ffmpeg. + ffmpeg_node = transcoder_node.TranscoderNode(processed_inputs, + audio_outputs, + video_outputs, + input_config, + pipeline_config) + + self._nodes.append(ffmpeg_node) + + text_streams = [] + if has_text_input: + for i in input_config.inputs: + if i.has_text(): + text_streams.append(i) + + # Process input through a packager node using Shaka Packager. + package_node = packager_node.PackagerNode(audio_outputs, + video_outputs, + text_streams, + output_dir, + pipeline_config) + self._nodes.append(package_node) + + if bucket_url: + # Import the cloud node late, so that the cloud deps are optional. + from . import cloud_node + push_to_cloud = cloud_node.CloudNode(output_dir, bucket_url, + self._temp_dir) + self._nodes.append(push_to_cloud) + + for node in self._nodes: + node.start() + + def is_running(self): + """Return True if we have nodes and all of them are still running.""" + return self._nodes and all(n.is_running() for n in self._nodes) + + def stop(self): + """Stop all nodes.""" + for node in self._nodes: + node.stop() + self._nodes = [] + + def _add_audio(self, input, channels, codecs): + audio_outputs = [] + if input.has_audio(): + language = input.get_language() or self._probe_language(input) + for codec in codecs: + audio_outputs.append(metadata.Metadata(self._create_pipe(), + channels, audio_codec=codec, + lang=language)) + return audio_outputs + + def _add_video(self, input, resolutions, codecs): + video_outputs = [] + if input.has_video(): + for codec in codecs: + hardware_encoding_required = False + if codec.startswith('hw:'): + hardware_encoding_required = True + codec = codec.split(':')[1] + in_res = input.get_resolution() + for out_res in resolutions: + # Only going to output lower or equal resolution videos. + # Upscaling is costly and does not do anything. + if (metadata.RESOLUTION_MAP[in_res] >= + metadata.RESOLUTION_MAP[out_res]): + video_outputs.append(metadata.Metadata(self._create_pipe(), + res_string=out_res, video_codec=codec, + hardware=hardware_encoding_required)) + return video_outputs + + def _probe_language(self, input): + # ffprobe {input}: list out metadata of input + # -show_entries stream=index:stream_tags=language: list out tracks with + # stream and language information + # -select_streams {track}: Only return stream/language information for + # specified track. + # -of compact=p=0:nk=1: Specify no keys printed and don't print the name + # at the beginning of each line. + command = ['ffprobe', input.get_name(), '-show_entries', + 'stream=index:stream_tags=language', '-select_streams', + str(input.get_track()), '-of', 'compact=p=0:nk=1'] + + lang_str = subprocess.check_output(command).decode('utf-8') + # The regex is looking for a string that is of the format number|language. + # Once it finds a number| match, it will copy the string until the end of + # the line. + lang_match = re.search(r'\d+\|(.*$)', lang_str) + if lang_match: + return lang_match.group(1) + + def is_vod(self): + return self.pipeline_config.mode == 'vod' + +def check_version(name, command, minimum_version): + min_version_string = '.'.join([str(x) for x in minimum_version]) + + try: + version_string = str(subprocess.check_output(command)) + except: + raise FileNotFoundError(name + ' not installed! Please install version ' + + min_version_string + ' or higher of ' + name + '.') + + version_match = re.search(r'([0-9]+)\.([0-9]+)\.([0-9]+)', version_string) + + if version_match == None: + raise VersionError(name + ' version not found in string output!') + + version = (int(version_match.group(1)), int(version_match.group(2))) + if version < minimum_version: + raise VersionError(name + ' not installed! Please install version ' + + min_version_string + ' or higher of ' + name + '.') diff --git a/streamer/default_config.py b/streamer/default_config.py new file mode 100644 index 0000000..9490369 --- /dev/null +++ b/streamer/default_config.py @@ -0,0 +1,115 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A file with the default configs.""" + +import base64 +import os + +from . import metadata + +INPUT_DEFAULT_CONFIG = { + # List of inputs. Each one is a dictionary. + 'inputs': [ + { + # Name of the input. + 'name': 'test_assets/BigBuckBunny.1080p.mp4', + # Type of input. To be used only for live. Can be looped_file, raw_images + # or webcam. + 'input_type': 'looped_file', + # The type of media for the input. Can be audio or video. + 'media_type': 'video', + # Frame rate per second. + 'frame_rate': 24.0, + # The input resolution. + 'resolution': '1080p', + # The track number. + 'track_num': 0, + # Whether or not the video is interlaced. + 'is_interlaced': False, + # Language of the audio stream. + # TODO: Add different default config input entries for audio, video and + # text entries so that fields match up with each type of entry. + 'language': 'und', + }, + ], +} + +# Contains sets of valid values for certain fields in the inputs list. +INPUT_VALID_VALUES = { + 'input_type': {'raw_images', 'looped_file', 'webcam'}, + 'media_type': {'audio', 'video', 'text'}, +} + +OUTPUT_DEFAULT_CONFIG = { + # Mode of streaming. Can either be live or vod. + 'streaming_mode': 'live', + 'transcoder': { + # A list of resolutions to encode. + 'resolutions': [ + '720p', + '480p', + ], + # The number of audio channels to encode with. + 'channels': 2, + # The codecs to encode with. + 'audio_codecs': [ + 'aac', + ], + 'video_codecs': [ + 'h264', + ], + }, + 'packager': { + # Manifest format (dash, hls). + 'manifest_format': [ + 'dash', + 'hls', + ], + # Length of each segment in seconds. + 'segment_size': 10, + # Forces the use of SegmentTemplate in DASH. + 'segment_per_file': True, + 'encryption': { + # Enables encryption. + # If disabled, the following settings are ignored. + 'enable': False, + # Content identifier that identifies which encryption key to use. + 'content_id': base64.b16encode(os.urandom(16)).decode('UTF-8'), + # Key server url. An encryption key is generated from this server. + 'key_server_url': 'https://license.uat.widevine.com/cenc/getcontentkey/widevine_test', + # The name of the signer. + 'signer': 'widevine_test', + # AES signing key in hex string. + 'signing_key': '1ae8ccd0e7985cc0b6203a55855a1034afc252980e970ca90e5202689f947ab9', + # AES signing iv in hex string. + 'signing_iv': 'd58ce954203b7c9a9a9d467f59839249', + # Protection scheme (cenc or cbcs) + # These are different methods of using a block cipher to encrypt media. + 'protection_scheme': 'cenc', + # Seconds of unencrypted media at the beginning of the stream. + 'clear_lead': 10, + }, + }, +} + +# Contains sets of valid values for certain fields in the output config. +OUTPUT_VALID_VALUES = { + 'streaming_mode': {'live', 'vod'}, + 'manifest_format': {'dash', 'hls'}, + 'protection_scheme': {'cenc', 'cbcs'}, + 'resolutions': metadata.RESOLUTION_MAP.keys(), + 'audio_codecs': {'aac', 'opus'}, + 'video_codecs': {'h264', 'vp9', 'hw:h264', 'hw:vp9'}, +} diff --git a/streamer/discard_node.py b/streamer/discard_node.py new file mode 100644 index 0000000..56eecdf --- /dev/null +++ b/streamer/discard_node.py @@ -0,0 +1,34 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module sending content in named pipes to /dev/null.""" + +import os +import shlex +import subprocess + +from . import node_base + +class DiscardNode(node_base.NodeBase): + + def __init__(self, named_pipes): + node_base.NodeBase.__init__(self) + self._named_pipes = named_pipes + + def start(self): + # Tail allows reading from multiple sources simultaneously, so it can + # concurrently read from all the pipes passed into named_pipes. + cmd = 'tail -f %s >/dev/null' % ' '.join( + map(shlex.quote, self._named_pipes)) + self._process = subprocess.Popen(cmd, shell=True) diff --git a/streamer/input_configuration.py b/streamer/input_configuration.py new file mode 100644 index 0000000..fe0450c --- /dev/null +++ b/streamer/input_configuration.py @@ -0,0 +1,127 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that organizes the input configs.""" + +from . import default_config +from . import validation + +INPUTS = 'inputs' + +class InputConfig(): + + def __init__(self, user_config, + default_config = default_config.INPUT_DEFAULT_CONFIG, + valid_values = default_config.INPUT_VALID_VALUES): + validation.setup_config(user_config, default_config, valid_values) + self.dict = user_config + self.inputs = [Input(i) for i in self.dict[INPUTS]] + +class Input(): + + def __init__(self, input): + self._input = input + + def get_name(self): + return self._input['name'] + + def get_input_type(self): + if 'input_type' in self._input: + return self._input['input_type'] + return None + + def get_media_type(self): + return self._input['media_type'] + + def get_frame_rate(self): + return self._input['frame_rate'] + + def get_resolution(self): + return self._input['resolution'] + + def get_track(self): + if 'track_num' in self._input: + return self._input['track_num'] + return 0 + + def get_interlaced(self): + if 'is_interlaced' in self._input: + return self._input['is_interlaced'] + return False + + def get_language(self): + if 'language' in self._input: + return self._input['language'] + return None + + def has_video(self): + if 'input_type' in self._input: + if (self._input['input_type'] == 'webcam' or + self._input['input_type'] == 'raw_images'): + return True + if 'media_type' in self._input: + if self._input['media_type'] == 'video': + return True + return False + + def has_audio(self): + if 'media_type' in self._input: + if self._input['media_type'] == 'audio': + return True + return False + + def has_text(self): + if 'media_type' in self._input: + if self._input['media_type'] == 'text': + return True + return False + + def check_text_entry(self): + if 'language' not in self._input: + raise RuntimeError('language must be specified for text track') + + def check_input_entry(self): + if 'name' not in self._input: + raise RuntimeError('name field must be in dictionary entry!') + elif 'media_type' not in self._input: + if (self._input['input_type'] != 'webcam' and + self._input['input_type'] != 'raw_images'): + raise RuntimeError('media_type field must be in dictionary entry!') + + def check_video_entry(self): + if 'frame_rate' not in self._input: + raise RuntimeError('frame_rate field must be in video dictionary entry!') + elif 'resolution' not in self._input: + raise RuntimeError('resolution field must be in video dictionary entry!') + + def check_live_validity(self): + if self.has_text(): + self.check_text_entry() + return + if self.has_video(): + self.check_video_entry() + self.check_input_entry() + if 'input_type' not in self._input: + raise RuntimeError('input_type field must be in dictionary entry!') + + def check_vod_validity(self): + if self.has_text(): + self.check_text_entry() + return + if self.has_video(): + self.check_video_entry() + self.check_input_entry() + if 'track_num' not in self._input: + raise RuntimeError('track_num field must be in dictionary entry!') + diff --git a/streamer/loop_input_node.py b/streamer/loop_input_node.py new file mode 100644 index 0000000..24b5902 --- /dev/null +++ b/streamer/loop_input_node.py @@ -0,0 +1,68 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that uses ffmpeg to loop a local file into a named pipe.""" + +from . import node_base + +class LoopInputNode(node_base.NodeBase): + def __init__(self, input_path, output_path, downmix_to_stereo=True): + node_base.NodeBase.__init__(self) + self._input_path = input_path + self._output_path = output_path + self._downmix_to_stereo = downmix_to_stereo + + def start(self): + args = [ + 'ffmpeg', + # Loop the input forever. + '-stream_loop', '-1', + # Read input in real time. + '-re', + # Suppresses all messages except warnings and errors. + '-loglevel', 'warning', + # The input itself. + '-i', self._input_path, + # Format the output as MPEG2-TS, which works well in a pipe. + '-f', 'mpegts', + # Copy the video stream directly. + '-c:v', 'copy', + ] + + # FIXME: 5.1 surround sound in TS, as output by ffmpeg, is rejected by + # Shaka Packager. https://github.com/google/shaka-packager/issues/598 + if self._downmix_to_stereo: + args += [ + # Re-encode audio as AAC at 192kbit. + '-c:a', 'aac', '-b:a', '192k', + # Downmix to 2 channels (stereo). + '-ac', '2', + ] + else: + args += [ + # Copy the audio stream directly. + '-c:a', 'copy', + ] + + args += [ + # Do not prompt for output files that already exist. Since we created + # the named pipe in advance, it definitely already exists. A prompt + # would block ffmpeg to wait for user input. + '-y', + # The output itself. + self._output_path, + ] + + self._process = self._create_process(args) + diff --git a/streamer/metadata.py b/streamer/metadata.py new file mode 100644 index 0000000..cb0b571 --- /dev/null +++ b/streamer/metadata.py @@ -0,0 +1,81 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that maps channels to its respective bitrate and resolutions to its +respective height, bitrate and profile.""" + +class ChannelData(): + + def __init__(self, aac_bitrate, opus_bitrate): + self.aac_bitrate = aac_bitrate + self.opus_bitrate = opus_bitrate + +class ResolutionData(): + + def __init__(self, height, h264_bitrate, vp9_bitrate, h264_profile): + self.height = height + self.h264_bitrate = h264_bitrate + self.vp9_bitrate = vp9_bitrate + self.h264_profile = h264_profile + + def __eq__(self, other): + return self.height == other.height + + def __ne__(self, other): + return not self.__eq__(other) + + def __ge__(self, other): + return self.height >= other.height + +# A map of channels to ChannelData objects which contains the AAC and Opus +# bitrate information of a given channel. +CHANNEL_MAP = { + 2: ChannelData(128, 64), + 6: ChannelData(192, 96), +} + +# A map of resolutions to ResolutionData objects which contain +# the height and H264 bitrate of a given resolution. +RESOLUTION_MAP = { + '144p': ResolutionData(144, '108k', '95k', 'baseline'), + '240p': ResolutionData(240, '242k', '150k', 'main'), + '360p': ResolutionData(360, '400k', '276k', 'main'), + '480p': ResolutionData(480, '2M', '750k', 'main'), + '576p': ResolutionData(576, '2.5M', '1M', 'main'), + '720p': ResolutionData(720, '3M', '2M', 'main'), + '720p-hfr': ResolutionData(720, '4M', '4M', 'main'), + '1080p': ResolutionData(1080, '5M', '4M', 'high'), + '1080p-hfr': ResolutionData(1080, '6M', '6M', 'high'), + '2k': ResolutionData(1440, '9M', '6M', 'high'), + '2k-hfr': ResolutionData(1440, '14M', '9M', 'high'), + '4k': ResolutionData(2160, '17M', '12M', 'uhd'), + '4k-hfr': ResolutionData(2160, '25M', '18M', 'uhd'), +} + +class Metadata(): + + def __init__(self, pipe, channels = None, res_string = None, + audio_codec = None, video_codec = None, lang=None, + hardware=None): + self.pipe = pipe + if channels: + self.channels = channels + self.audio_codec = audio_codec + self.channel_data = CHANNEL_MAP[channels] + self.lang = lang + if res_string: + self.res = res_string + self.video_codec = video_codec + self.resolution_data = RESOLUTION_MAP[res_string] + self.hardware = hardware diff --git a/streamer/node_base.py b/streamer/node_base.py new file mode 100644 index 0000000..38b9b5a --- /dev/null +++ b/streamer/node_base.py @@ -0,0 +1,76 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A base class for nodes that run a single subprocess.""" + +import abc +import shlex +import subprocess +import time + +class NodeBase(object): + @abc.abstractmethod + def __init__(self): + self._process = None + + @abc.abstractmethod + def start(self): + """Start the subprocess. + + Should be overridden by the subclass to construct a command line, call + self._create_process, and assign the result to self._process. + """ + pass + + def _create_process(self, args): + """A central point to create subprocesses, so that we can debug the + command-line arguments. + + Args: + args: An array of strings, the command line of the subprocess. + Returns: + The Popen object of the subprocess. + """ + # Print arguments formatted as output from bash -x would be. + # This makes it easy to see the arguments and easy to copy/paste them for + # debugging in a shell. + print('+ ' + ' '.join([shlex.quote(arg) for arg in args])) + return subprocess.Popen(args, stdin = subprocess.DEVNULL) + + def is_running(self): + """Returns True if the subprocess is still running, and False otherwise.""" + if not self._process: + return False + + self._process.poll() + if self._process.returncode is not None: + return False + + return True + + def stop(self): + """Stop the subprocess if it's still running.""" + if self._process: + # Slightly more polite than kill. Try this first. + + self._process.terminate() + + if self.is_running(): + # If it's not dead yet, wait 1 second. + time.sleep(1) + + if self.is_running(): + # If it's still not dead, use kill. + self._process.kill() + # Don't wait to see the results. If this didn't work, nothing will. diff --git a/streamer/packager_node.py b/streamer/packager_node.py new file mode 100644 index 0000000..6803821 --- /dev/null +++ b/streamer/packager_node.py @@ -0,0 +1,236 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that feeds information from two named pipes into shaka-packager.""" + +from . import metadata +from . import node_base + +MP4_AUDIO_INIT_SEGMENT = '{dir}/audio_{channels}_{bitrate}_init.mp4' +WEBM_AUDIO_INIT_SEGMENT = '{dir}/audio_{channels}_{bitrate}_init.webm' +MP4_AUDIO_SEGMENT_TEMPLATE = '{dir}/audio_{channels}_{bitrate}_$Number$.m4s' +WEBM_AUDIO_SEGMENT_TEMPLATE = '{dir}/audio_{channels}_{bitrate}_$Number$.webm' +MP4_AUDIO_OUTPUT = '{dir}/audio_{channels}_{bitrate}_output.mp4' +WEBM_AUDIO_OUTPUT = '{dir}/audio_{channels}_{bitrate}_output.webm' +MP4_VIDEO_INIT_SEGMENT = '{dir}/video_{res}_{bitrate}_init.mp4' +WEBM_VIDEO_INIT_SEGMENT = '{dir}/video_{res}_{bitrate}_init.webm' +MP4_VIDEO_SEGMENT_TEMPLATE = '{dir}/video_{res}_{bitrate}_$Number$.m4s' +WEBM_VIDEO_SEGMENT_TEMPLATE = '{dir}/video_{res}_{bitrate}_$Number$.webm' +MP4_VIDEO_OUTPUT = '{dir}/video_{res}_{bitrate}_output.mp4' +WEBM_VIDEO_OUTPUT = '{dir}/video_{res}_{bitrate}_output.webm' + +TEXT_INIT_SEGMENT = '{dir}/text_{lang}_init.mp4' +TEXT_SEGMENT_TEMPLATE = '{dir}/text_{lang}_$Number$.m4s' +TEXT_OUTPUT = '{dir}/text_{lang}_output.mp4' + +DASH_OUTPUT = '/output.mpd' +HLS_OUTPUT = '/master_playlist.m3u8' + +class SegmentError(Exception): + """Raise when segment is incompatible with format.""" + pass + +class PackagerNode(node_base.NodeBase): + + def __init__(self, audio_inputs, video_inputs, text_inputs, output_dir, config): + node_base.NodeBase.__init__(self) + self._audio_inputs = audio_inputs + self._video_inputs = video_inputs + self._text_inputs = text_inputs + self._output_dir = output_dir + self._config = config + + def start(self): + args = [ + 'packager', + ] + + for input in self._audio_inputs: + audio_dict = {'in': input.pipe, 'stream': 'audio'} + if input.lang != 'und': + audio_dict['language'] = input.lang + args += self._create_audio(audio_dict, input) + + for input in self._video_inputs: + video_dict = {'in': input.pipe, 'stream': 'video'} + args += self._create_video(video_dict, input) + + for input in self._text_inputs: + text_dict = { + 'in': input.get_name(), + 'stream': 'text', + 'language': input.get_language(), + } + args += self._create_text(text_dict, input.get_language()) + + args += [ + # Segment duration given in seconds. + '--segment_duration', str(self._config.packager['segment_size']), + ] + + if self._config.mode == 'live': + args += [ + # Number of seconds the user can rewind through backwards. + '--time_shift_buffer_depth', '300', + # Number of segments preserved outside the current live window. + '--preserved_segments_outside_live_window', '1', + ] + + args += self._setup_manifest_format() + + args += [ + # use an IO block size of ~65K for a threaded IO file. + '--io_block_size', '65536', + ] + + if self._config.encryption['enable']: + args += self._setup_encryption() + + self._process = self._create_process(args) + + def _create_text(self, text_dict, language): + if self._config.packager['segment_per_file']: + text_dict['init_segment'] = (TEXT_INIT_SEGMENT. + format(dir=self._output_dir, lang=language)) + text_dict['segment_template'] = (TEXT_SEGMENT_TEMPLATE. + format(dir=self._output_dir, lang=language)) + else: + text_dict['output'] = (TEXT_OUTPUT. + format(dir=self._output_dir, lang=language)) + return [_packager_stream_arg(text_dict)] + + def _create_audio(self, dict, audio): + if self._config.packager['segment_per_file']: + if audio.audio_codec == 'aac': + self._setup_segmented_output(dict, MP4_AUDIO_INIT_SEGMENT, + MP4_AUDIO_SEGMENT_TEMPLATE, 'channels', audio.channels, + metadata.CHANNEL_MAP[audio.channels].aac_bitrate) + elif audio.audio_codec == 'opus': + self._setup_segmented_output(dict, WEBM_AUDIO_INIT_SEGMENT, + WEBM_AUDIO_SEGMENT_TEMPLATE, 'channels', audio.channels, + metadata.CHANNEL_MAP[audio.channels].opus_bitrate) + else: + if self._config.mode == 'vod': + if audio.audio_codec == 'aac': + self._setup_single_file_output(dict, MP4_AUDIO_OUTPUT, 'channels', + audio.channels, metadata.CHANNEL_MAP[audio.channels].aac_bitrate) + elif audio.audio_codec == 'opus': + self._setup_single_file_output(dict, MP4_AUDIO_OUTPUT, 'channels', + audio.channels, metadata.CHANNEL_MAP[audio.channels].opus_bitrate) + else: + # Live mode doesn't support a non-segment video. + raise SegmentError('Non segment does not work with LIVE') + return [_packager_stream_arg(dict)] + + def _create_video(self, dict, video): + if self._config.packager['segment_per_file']: + if video.video_codec == 'h264': + self._setup_segmented_output(dict, MP4_VIDEO_INIT_SEGMENT, + MP4_VIDEO_SEGMENT_TEMPLATE, 'res', video.res, + video.resolution_data.h264_bitrate) + elif video.video_codec == 'vp9': + self._setup_segmented_output(dict, WEBM_VIDEO_INIT_SEGMENT, + WEBM_VIDEO_SEGMENT_TEMPLATE, 'res', video.res, + video.resolution_data.vp9_bitrate) + else: + if self._config.mode == 'vod': + if video.video_codec == 'h264': + self._setup_single_file_output(dict, MP4_VIDEO_OUTPUT, 'res', + video.res, metadata.RESOLUTION_MAP[video.res].h264_bitrate) + elif video.video_codec == 'vp9': + self._setup_single_file_output(dict, WEBM_VIDEO_OUTPUT, 'res', + video.res, metadata.RESOLUTION_MAP[video.res].vp9_bitrate) + else: + raise SegmentError("Non segment does not work with LIVE") + return [_packager_stream_arg(dict)] + + def _setup_segmented_output(self, dict, init_segment_name, segment_name, + channels_or_res, channel_or_res_info, + bitrate_info): + if channels_or_res == 'channels': + # Set the initial segment. + dict['init_segment'] = (init_segment_name. + format(dir=self._output_dir, channels=channel_or_res_info, + bitrate=bitrate_info)) + # Create the individual segments. + dict['segment_template'] = (segment_name. + format(dir=self._output_dir, channels=channel_or_res_info, + bitrate=bitrate_info)) + elif channels_or_res == 'res': + dict['init_segment'] = (init_segment_name. + format(dir=self._output_dir, res=channel_or_res_info, + bitrate=bitrate_info)) + dict['segment_template'] = (segment_name. + format(dir=self._output_dir, res=channel_or_res_info, + bitrate=bitrate_info)) + + def _setup_single_file_output(self, dict, file_name, channels_or_res, + channel_or_res_info, bitrate_info): + if channels_or_res == 'channels': + dict['output'] = (file_name.format(dir=self._output_dir, + channels=channel_or_res_info, + bitrate=bitrate_info)) + elif channels_or_res == 'res': + dict['output'] = (file_name.format(dir=self._output_dir, + res=channel_or_res_info, + bitrate=bitrate_info)) + + def _setup_manifest_format(self): + args = [] + if 'dash' in self._config.packager['manifest_format']: + if self._config.mode == 'vod': + args += [ + '--generate_static_mpd', + ] + args += [ + # Generate DASH manifest file. + '--mpd_output', self._output_dir + DASH_OUTPUT, + ] + if 'hls' in self._config.packager['manifest_format']: + args += [ + # Generate HLS manifest file. + '--hls_master_playlist_output', + self._output_dir + HLS_OUTPUT, + ] + if self._config.mode == 'live': + args += [ + '--hls_playlist_type', 'LIVE', + ] + else: + args += [ + '--hls_playlist_type', 'VOD', + ] + return args + + def _setup_encryption(self): + # Sets up encryption of content. + args = [ + '--enable_widevine_encryption', + '--key_server_url', self._config.encryption['key_server_url'], + '--content_id', self._config.encryption['content_id'], + '--signer', self._config.encryption['signer'], + '--aes_signing_key', self._config.encryption['signing_key'], + '--aes_signing_iv', self._config.encryption['signing_iv'], + '--protection_scheme', self._config.encryption['protection_scheme'], + '--clear_lead', str(self._config.encryption['clear_lead']), + ] + return args + + +def _packager_stream_arg(opts): + ret = '' + for key, value in opts.items(): + ret += key + '=' + value + ',' + return ret + diff --git a/streamer/pipeline_configuration.py b/streamer/pipeline_configuration.py new file mode 100644 index 0000000..cfca778 --- /dev/null +++ b/streamer/pipeline_configuration.py @@ -0,0 +1,35 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that organizes the pipeline config.""" + +from . import default_config +from . import validation + +MODE = 'streaming_mode' +TRANSCODER = 'transcoder' +PACKAGER = 'packager' +ENCRYPTION = 'encryption' + +class PipelineConfig(): + + def __init__(self, user_config, + default_config = default_config.OUTPUT_DEFAULT_CONFIG, + valid_values = default_config.OUTPUT_VALID_VALUES): + validation.setup_config(user_config, default_config, valid_values) + self.dict = user_config + self.mode = self.dict[MODE] + self.transcoder = self.dict[TRANSCODER] + self.packager = self.dict[PACKAGER] + self.encryption = self.packager[ENCRYPTION] diff --git a/streamer/transcoder_node.py b/streamer/transcoder_node.py new file mode 100644 index 0000000..af2a076 --- /dev/null +++ b/streamer/transcoder_node.py @@ -0,0 +1,261 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that pushes input to ffmpeg to transcode into various formats.""" + +from . import input_configuration +from . import metadata +from . import node_base + +# For H264, there are different profiles with different required command line +# arguments. +profile_args = { + 'baseline': ['-profile:v', 'baseline', '-level:v', '3.0'], + 'main': ['-profile:v', 'main', '-level:v', '3.1'], + 'high': ['-profile:v', 'high', '-level:v', '4.0'], + 'uhd': ['-profile:v', 'high', '-level:v', '5.1'], +} + +class TranscoderNode(node_base.NodeBase): + + def __init__(self, inputs, output_audios, output_videos, input_config, config): + node_base.NodeBase.__init__(self) + self._inputs = inputs + self._output_audios = output_audios + self._output_videos = output_videos + self._input_config = input_config + self._config = config + + def start(self): + args = [ + 'ffmpeg', + # Do not prompt for output files that already exist. Since we created + # the named pipe in advance, it definitely already exists. A prompt + # would block ffmpeg to wait for user input. + '-y', + ] + + for i in range(len(self._inputs)): + input = self._input_config.inputs[i] + + if self._config.mode == 'live': + if any([output.hardware for output in self._output_videos]): + args += [ + # Hardware acceleration args. + '-hwaccel', 'vaapi', + '-vaapi_device', '/dev/dri/renderD128', + ] + args += self._live_input(input, self._inputs[i]) + + elif self._config.mode == 'vod': + args += [ + # The input itself. + '-i', input.get_name(), + ] + + # Check if the media type of the input is audio and if there are expected + # outputs for the audio input. + if input.get_media_type() == 'audio' and self._output_audios: + for audio in self._output_audios: + if self._config.mode == 'vod': + args += [ + # Map corresponding stream to output file. + '-map', '0:{0}'.format(input.get_track()), + ] + args += self._encode_audio(audio, audio.audio_codec, audio.channels, + metadata.CHANNEL_MAP[audio.channels]) + + # Check if the media type of the input is video and if there are expected + # outputs for the video input. + if input.get_media_type() == 'video' and self._output_videos: + for video in self._output_videos: + if self._config.mode == 'vod': + args += [ + # Map corresponding stream to output file. + '-map', '0:{0}'.format(input.get_track()), + ] + group_of_pictures = int(self._config.packager['segment_size'] * + input.get_frame_rate()) + args += self._encode_video(video, video.video_codec, + group_of_pictures, + metadata.RESOLUTION_MAP[video.res], + input.get_frame_rate(), + input.get_interlaced()) + + self._process = self._create_process(args) + + def _live_input(self, input_object, input_path): + args = [] + if input_object.get_input_type() == 'looped_file': + pass + elif input_object.get_input_type() == 'raw_images': + args += [ + # Format the input as a stream of images fed into a pipe. + '-f', 'image2pipe', + # Assumes the images are in ppm raw format. + '-vcodec', 'ppm', + # Set the frame rate to the one specified in the input config. + '-r', input_object.get_frame_rate(), + ] + elif input_object.get_input_type() == 'webcam': + args += [ + # Format the input using the webcam format. + '-f', 'video4linux2', + ] + args += [ + # The input itself. + '-i', input_path, + ] + return args + + def _encode_audio(self, audio, codec, channels, channel_map): + args = [ + # No video encoding for audio. + '-vn', + # Set the number of channels to the one specified in the VOD config + # file. + '-ac', str(channels), + ] + if codec == 'aac': + args += [ + # Format with MPEG-TS for a pipe. + '-f', 'mpegts', + # AAC audio codec. + '-c:a', 'aac', + # Set bitrate to the one specified in the VOD config file. + '-b:a', '{0}k'.format(channel_map.aac_bitrate), + # Reorder mp4 segments so that the moov segment is at the front. + # Needed for output to a pipe. + '-movflags', '+faststart', + ] + elif codec == 'opus': + args += [ + # Opus encoding has output format webm. + '-f', 'webm', + # Opus audio codec. + '-c:a', 'libopus', + # Set bitrate to the one specified in the VOD config file. + '-b:a', '{0}k'.format(channel_map.opus_bitrate), + # DASH-compatible output format. + '-dash', '1', + ] + args += [ + # The output. + audio.pipe, + ] + return args + + def _encode_video(self, video, codec, gop_size, res_map, frame_rate, + is_interlaced): + filters = [] + args = [ + # No audio encoding for video. + '-an', + # Full pelME compare function. + '-cmp', 'chroma', + ] + #TODO: auto detection of interlacing + if is_interlaced: + # Sanity check: since interlaced files are made up of two interlaced + # frames, the frame rate must be even and not too small. + assert frame_rate % 2 == 0 and frame_rate >= 48 + filters.append('pp=fd') + args.extend(['-r', str(frame_rate / 2)]) + + if video.hardware: + filters.append('format=nv12') + filters.append('hwupload') + filters.append('scale_vaapi={0}:{1}'.format(-2, res_map.height)) + else: + filters.append('scale={0}:{1}'.format(-2, res_map.height)) + + if codec == 'h264': + args += [ + # MPEG-TS format works well in a pipe. + '-f', 'mpegts', + ] + + if self._config.mode == 'live': + args += [ + # Encodes with highest-speed presets for real-time live streaming. + '-preset', 'ultrafast', + ] + else: + args += [ + # Take your time for VOD streams. + '-preset', 'slow', + # Apply the loop filter for higher quality output. + '-flags', '+loop', + ] + + if video.hardware: + args += [ + # H264 VAAPI video codec. + '-c:v', 'h264_vaapi', + ] + else: + args += [ + # H264 video codec. + '-c:v', 'h264', + ] + + args += [ + # Set bitrate to the one specified in the VOD config file. + '-b:v', '{0}'.format(res_map.h264_bitrate), + # Set maximum number of B frames between non-B frames. + '-bf', '0', + # Reorder mp4 segments so that the moov segment is at the front. + # Needed for output to a pipe. + '-movflags', '+faststart', + # The only format supported by QT/Apple. + '-pix_fmt', 'yuv420p', + # Require a closed GOP. Some decoders don't support open GOPs. + '-flags', '+cgop', + ] + # Use different ffmpeg options depending on the H264 profile. + args += profile_args[res_map.h264_profile] + + elif codec == 'vp9': + args += [ + # Format using webm. + '-f', 'webm', + ] + + if video.hardware: + args += [ + # VP9 VAAPI video codec. + '-c:v', 'vp9_vaapi', + ] + else: + args += [ + # VP9 video codec. + '-c:v', 'vp9', + ] + + args += [ + # Set bitrate to the one specified in the VOD config file. + '-b:v', '{0}'.format(res_map.vp9_bitrate), + # DASH-compatible output format. + '-dash', '1', + ] + + args += [ + # Set minimum and maximum GOP length. + '-keyint_min', str(gop_size), '-g', str(gop_size), + # Set video filters. + '-vf', ','.join(filters), + # The output. + video.pipe, + ] + return args diff --git a/streamer/validation.py b/streamer/validation.py new file mode 100644 index 0000000..5c1a51c --- /dev/null +++ b/streamer/validation.py @@ -0,0 +1,120 @@ +# Copyright 2019 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +"""A module that validates the config file. +It also fills in default values in empty fields.""" + +def _assert_valid_keys(user_config, default_config): + """Checks if user config keys are valid + + Raises: + KeyError: If user_config key does not exist in default_config. + """ + invalid_config = set(user_config.keys()) - set(default_config.keys()) + if invalid_config: + invalid_args = ', '.join(invalid_config) + raise KeyError( + "These are unrecognized field(s) in your config: {}".format( + invalid_args)) + +def _assert_valid_type(user_value, default_val, key): + """Type checks user config to defaults. + + Raises: + TypeError: If user_value type is different from the default_value type. + """ + config_type = type(user_value) + default_type = type(default_val) + + if config_type is int and default_type is float: + config_type = float + + if config_type is not default_type: + raise TypeError( + "The field '{}' has a {} value when it should be a {}".format( + key, + config_type, + default_type)) + +def _assert_valid_value(user_value, valid_values, key): + if key in valid_values: + valid_set = valid_values[key] + # Value checking every item in the user config list is valid. + if type(user_value) is list and set(user_value) - valid_set: + raise ValueError( + "The field '{}' has a value {} which is not one of {}".format( + key, + set(user_value) - valid_set, + valid_set)) + # Value checking user config item is valid. + elif type(user_value) is not list and user_value not in valid_set: + raise ValueError( + "The field '{}' has a value {} which is not one of {}".format( + key, + user_value, + valid_set)) + +def _set_defaults(user_config, default_key, default_val): + """Sets the user config to default if value not set. + + Args: + user_config: A dict containing the passed config values. + default_key: The key from the default_config. + default_val: The value corresponding to the default_key in default_config. + Returns: + A boolean saying if a default value was set or not. + Modifies user_config if default key not found in user_config yet. + """ + if default_key not in user_config.keys(): + user_config[default_key] = default_val + return True + return False + +def setup_config(user_config, default_config, valid_values): + """Validates a given config dict then combines with defaults. + + Args: + user_config: A dict containing the passed config values. + default_config: A dict containing default config values. + + Returns: + Nothing. It modifies the user_config it is given. + """ + _assert_valid_keys(user_config, default_config) + + # I am going to iterate over the defaults to change the user config instead. + # Although it will be not as clean as .update(config), I think it will be + # better to have a read-only dict since .update(config) modifies the default. + for key, default_val in default_config.items(): + if _set_defaults(user_config, key, default_val): + continue + + _assert_valid_type(user_config[key], default_val, key) + _assert_valid_value(user_config[key], valid_values, key) + + if type(user_config[key]) is list: + # If a configuration is an empty list, set it to equal the list in the + # default configuration. + if not user_config[key]: + user_config[key] = default_val + + for user_val in user_config[key]: + _assert_valid_type(user_val, default_val[0], key) + if type(user_val) is dict: + for key in user_val: + _assert_valid_type(user_val[key], default_val[0][key], key) + _assert_valid_value(user_val[key], valid_values, key) + + if type(default_val) is dict: + setup_config(user_config[key], default_val, valid_values) diff --git a/tests/karma.conf.js b/tests/karma.conf.js new file mode 100644 index 0000000..07fdae9 --- /dev/null +++ b/tests/karma.conf.js @@ -0,0 +1,32 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +/** @param {!Object} config */ +module.exports = function(config) { + config.set({ + basePath: __dirname, + browserNoActivityTimeout: 5 * 60 * 1000, // Disconnect after 5m silence + client: { + captureConsole: true, + }, + frameworks: ['jasmine'], + files: [ + // Shaka Player + '../node_modules/shaka-player/dist/shaka-player.compiled.js', + + // End to end tests + 'tests.js', + ], + }); +}; diff --git a/tests/tests.js b/tests/tests.js new file mode 100644 index 0000000..4ca637b --- /dev/null +++ b/tests/tests.js @@ -0,0 +1,465 @@ +// Copyright 2019 Google LLC +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// https://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +const flaskServerUrl = 'http://localhost:5000/'; +const dashManifestUrl = flaskServerUrl + 'output_files/output.mpd'; +const hlsManifestUrl = flaskServerUrl + 'output_files/master_playlist.m3u8'; +const TEST_DIR = 'test_assets/'; +let player; + +async function startStreamer(inputConfig, pipelineConfig) { + // Send a request to flask server to start Shaka Streamer. + const response = await fetch(flaskServerUrl + 'start', { + method: 'POST', + headers: { + 'Content-Type': 'text/plain', + }, + body: JSON.stringify({ + 'input_config': inputConfig, + 'pipeline_config': pipelineConfig + }), + }); + + if (!response.ok) { + throw new Error('Failed to produce manifest'); + } +} + +async function stopStreamer() { + // Send a request to flask server to stop Shaka Streamer. + const response = await fetch(flaskServerUrl + 'stop'); + if (!response.ok) { + throw new Error('Failed to close Shaka Streamer'); + } +} + +describe('Shaka Streamer', () => { + let video; + + beforeAll(() => { + shaka.polyfill.installAll(); + jasmine.DEFAULT_TIMEOUT_INTERVAL = 400 * 1000; + }); + + beforeEach(() => { + video = document.createElement('video'); + video.muted = true; + document.body.appendChild(video); + + player = new shaka.Player(video); + player.addEventListener('error', (error) => { + fail(error); + }); + }); + + afterEach(async () => { + await player.destroy(); + await stopStreamer(); + document.body.removeChild(video); + }); + + resolutionTests(hlsManifestUrl, '(hls)'); + resolutionTests(dashManifestUrl, '(dash)'); + liveTests(hlsManifestUrl, '(hls)'); + liveTests(dashManifestUrl, '(dash)'); + drmTests(hlsManifestUrl, '(hls)'); + drmTests(dashManifestUrl, '(dash)'); + codecTests(hlsManifestUrl, '(hls)'); + codecTests(dashManifestUrl, '(dash)'); + autoLanguageTests(hlsManifestUrl, '(hls)'); + autoLanguageTests(dashManifestUrl, '(dash)'); + languageTests(hlsManifestUrl, '(hls)'); + languageTests(dashManifestUrl, '(dash)'); + // TODO: Test is commented out until Packager outputs codecs for vtt in mp4. + // textTracksTests(hlsManifestUrl, '(hls)'); + textTracksTests(dashManifestUrl, '(dash)'); + vodTests(hlsManifestUrl, '(hls)'); + vodTests(dashManifestUrl, '(dash)'); + // TODO: Redo this test with both 5.1 and stereo once 5.1 is supported. + channelsTests(hlsManifestUrl, '(hls)'); + channelsTests(dashManifestUrl, '(dash)'); +}); + +function resolutionTests(manifestUrl, format) { + it('has output resolutions matching the resolutions in config ' + format, + async () => { + const inputConfigDict = { + 'inputs': [ + { + 'name': TEST_DIR + 'BigBuckBunny.1080p.mp4', + 'input_type': 'looped_file', + 'media_type': 'video', + 'frame_rate': 24.0, + 'resolution': '1080p', + 'track_num': 0, + }, + ] + }; + const pipelineConfigDict = { + // Mode of streaming. Can either be live or vod. + 'transcoder': { + // A list of resolutions to encode. + 'resolutions': [ + '4k', + '1080p', + '720p', + '480p', + '240p', + '144p', + ], + }, + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + + const trackList = player.getVariantTracks(); + const heightList = trackList.map(track => track.height); + heightList.sort((a, b) => a - b); + expect(heightList).toEqual([144, 240, 480, 720, 1080]); + }); +} + +function liveTests(manifestUrl, format) { + it('has a live streaming mode ' + format, async () => { + const inputConfigDict = { + 'inputs': [ + { + 'name': TEST_DIR + 'BigBuckBunny.1080p.mp4', + 'input_type': 'looped_file', + 'media_type': 'video', + 'frame_rate': 24.0, + 'resolution': '1080p', + 'track_num': 0, + }, + ] + }; + const pipelineConfigDict = { + // Mode of streaming. Can either be live or vod. + 'streaming_mode': 'live', + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + expect(player.isLive()).toBe(true); + }); +} + +function drmTests(manifestUrl, format) { + it('has encryption enabled ' + format, async () => { + const inputConfigDict = { + 'inputs': [ + { + 'name': TEST_DIR + 'BigBuckBunny.1080p.mp4', + 'input_type': 'looped_file', + 'media_type': 'video', + 'frame_rate': 24.0, + 'resolution': '1080p', + 'track_num': 0, + }, + ] + }; + const pipelineConfigDict = { + 'packager': { + 'encryption': { + // Enables encryption. + 'enable': true, + 'clear_lead': 0, + }, + }, + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + // Player should raise an error and not load because the media + // is encrypted and the player doesn't have a license server. + await expectAsync(player.load(manifestUrl)).toBeRejected( + "Encrypted media should not play without a license server"); + + player.configure({ + drm: { + servers: { + 'com.widevine.alpha': 'https://cwip-shaka-proxy.appspot.com/no_auth', + }, + }, + }); + // Player should now be able to load because the player has a license server. + await player.load(manifestUrl); + }); +} + +function codecTests(manifestUrl, format) { + it('has output codecs matching the codecs in config ' + format, async () => { + const inputConfigDict = { + 'inputs': [ + { + 'name': TEST_DIR + 'Sintel.2010.720p.Small.mkv', + 'media_type': 'video', + 'track_num': 0, + 'frame_rate': 24.0, + 'resolution': '4k', + 'is_interlaced': false, + }, + { + 'name': TEST_DIR + 'Sintel.2010.720p.Small.mkv', + 'media_type': 'audio', + 'track_num': 1, + }, + ], + }; + const pipelineConfigDict = { + 'transcoder': { + 'resolutions': [ + '144p', + ], + 'audio_codecs': [ + 'opus', + ], + 'video_codecs': [ + 'h264', + ], + }, + 'packager': { + 'manifest_format': [ + 'dash', + 'hls', + ], + }, + 'streaming_mode': 'vod', + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + + const trackList = player.getVariantTracks(); + const videoCodecList = trackList.map(track => track.videoCodec); + const audioCodecList = trackList.map(track => track.audioCodec); + expect(videoCodecList).toEqual(['avc1.42c01e']); + expect(audioCodecList).toEqual(['opus']); + }); +} + +function autoLanguageTests(manifestUrl, format) { + it('correctly autodetects the language embedded in the stream ' + format, + async () => { + // No language is specified in the input config, so the streamer will try + // to find the one embedded in the metadata. + const inputConfigDict = { + 'inputs': [ + { + 'name': TEST_DIR + 'Sintel.2010.720p.Small.mkv', + 'media_type': 'audio', + 'input_type': 'looped_file', + 'track_num': 1, + }, + ], + }; + const pipelineConfigDict = { + 'transcoder': { + 'resolutions': [ + '144p', + ], + }, + 'packager': { + 'manifest_format': [ + 'dash', + 'hls', + ], + }, + 'streaming_mode': 'live', + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + + const trackList = player.getVariantTracks(); + const lang = trackList.map(track => track.language); + expect(lang).toEqual(['en']); + }); +} + +function languageTests(manifestUrl, format) { + it('correctly sets the language read from the input config ' + format, + async() => { + const inputConfigDict = { + 'inputs': [ + { + 'name': TEST_DIR + 'Sintel.2010.720p.Small.mkv', + 'media_type': 'audio', + 'track_num': 1, + 'language': 'zh', + }, + ], + }; + const pipelineConfigDict = { + 'transcoder': { + 'resolutions': [ + '144p', + ], + }, + 'packager': { + 'manifest_format': [ + 'dash', + 'hls', + ], + }, + 'streaming_mode': 'vod', + + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + + const trackList = player.getVariantTracks(); + const lang = trackList.map(track => track.language); + expect(lang).toEqual(['zh']); + }); +} + +function textTracksTests(manifestUrl, format) { + it('outputs correct text tracks ' + format, async () => { + const inputConfigDict = { + // List of inputs. Each one is a dictionary. + 'inputs': [ + { + 'name': TEST_DIR + 'BigBuckBunny.1080p.mp4', + 'input_type': 'looped_file', + 'media_type': 'video', + 'frame_rate': 24.0, + 'resolution': '1080p', + 'track_num': 0, + }, + { + 'name': TEST_DIR + 'Sintel.2010.English.vtt', + 'media_type': 'text', + 'language': 'en', + }, + { + 'name': TEST_DIR + 'Sintel.2010.Spanish.vtt', + 'media_type': 'text', + 'language': 'es', + }, + { + 'name': TEST_DIR + 'Sintel.2010.Esperanto.vtt', + 'media_type': 'text', + 'language': 'eo', + }, + { + 'name': TEST_DIR + 'Sintel.2010.Arabic.vtt', + 'media_type': 'text', + 'language': 'ar', + }, + ], + }; + + const pipelineConfigDict = { + 'streaming_mode': 'vod', + 'transcoder': { + 'resolutions': [ + '144p', + ], + // This test currently only tests aac with h264, which means webm + // won't be an output. With webm, it doesnt work on chrome. + 'audio_codecs': [ + 'aac', + ], + 'video_codecs': [ + 'h264', + ], + }, + 'packager': { + 'segment_per_file': false, + }, + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + + const trackList = player.getTextTracks(); + const languageList = trackList.map(track => track.language); + languageList.sort(); + expect(languageList).toEqual(['ar', 'en', 'eo', 'es']); + }); +} + +function vodTests(manifestUrl, format) { + it('has a vod streaming mode', async () => { + const inputConfigDict = { + // List of inputs. Each one is a dictionary. + 'inputs': [ + { + 'name': TEST_DIR + 'BigBuckBunny.1080p.mp4', + 'input_type': 'looped_file', + 'media_type': 'video', + 'frame_rate': 24.0, + 'resolution': '1080p', + 'track_num': 0, + }, + ], + }; + + const pipelineConfigDict = { + 'streaming_mode': 'vod', + 'transcoder': { + 'resolutions': [ + '144p', + ], + 'audio_codecs': [ + 'aac', + ], + 'video_codecs': [ + 'h264', + ], + }, + 'packager': { + 'segment_per_file': false, + }, + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + expect(player.isLive()).toBe(false); + }); +} + +function channelsTests(manifestUrl, format) { + it('outputs the correct number of channels', async () => { + const inputConfigDict = { + // List of inputs. Each one is a dictionary. + 'inputs': [ + { + 'name': TEST_DIR + 'Sintel.2010.720p.Small.mkv', + 'input_type': 'looped_file', + 'media_type': 'audio', + 'track_num': 1, + }, + ], + }; + + const pipelineConfigDict = { + 'streaming_mode': 'vod', + 'transcoder': { + 'resolutions': [ + '144p', + ], + 'audio_codecs': [ + 'aac', + ], + 'video_codecs': [ + 'h264', + ], + 'channels': 2, + }, + 'packager': { + 'segment_per_file': false, + }, + }; + await startStreamer(inputConfigDict, pipelineConfigDict); + await player.load(manifestUrl); + const trackList = player.getVariantTracks(); + expect(trackList.length).toBe(1); + expect(trackList[0].channelsCount).toBe(2); + }); +}