Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support Inference with YOLOv8 Models #160

Merged
merged 8 commits into from
Jul 27, 2023
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
12 changes: 6 additions & 6 deletions .github/workflows/release-workflow.yml
Original file line number Diff line number Diff line change
Expand Up @@ -33,11 +33,11 @@ jobs:
shell: bash
run: |
if [ ${{ steps.extract_platform.outputs.platform }} = "windows" ]; then
echo "##[set-output name=name;]$(echo Portal Setup ${{ steps.extract_version.outputs.version }}.exe)"
echo "##[set-output name=name;]$(echo Portal Setup ${{ steps.extract_version.outputs.version }}.exe)"
elif [ ${{ steps.extract_platform.outputs.platform }} = "ubuntu" ]; then
echo "##[set-output name=name;]$(echo Portal-${{ steps.extract_version.outputs.version }}.AppImage)"
echo "##[set-output name=name;]$(echo Portal-${{ steps.extract_version.outputs.version }}.AppImage)"
else
echo "##[set-output name=name;]$(echo Portal-${{ steps.extract_version.outputs.version }}.dmg)"
echo "##[set-output name=name;]$(echo Portal-${{ steps.extract_version.outputs.version }}.dmg)"
fi
id: extract_executable_name

Expand All @@ -56,10 +56,10 @@ jobs:
working-directory: ./src/engine
shell: bash
run: |
pyinstaller -F run.py --hidden-import pydicom.encoders.gdcm --hidden-import pydicom.encoders.pylibjpeg --hidden-import engineio.async_drivers.threading --distpath ../../portal_build/dist
pyinstaller --collect-data ultralytics --hidden-import pydicom.encoders.gdcm --hidden-import pydicom.encoders.pylibjpeg --hidden-import engineio.async_drivers.threading -F run.py --distpath ../../portal_build/dist
if [ -f ../../portal_build/dist/run ]; then
chmod u+x ../../portal_build/dist/run
mv ../../portal_build/dist/run ../../portal_build/dist/run.exe
chmod u+x ../../portal_build/dist/run
mv ../../portal_build/dist/run ../../portal_build/dist/run.exe
fi
chmod u+x ../../portal_build/dist/run.exe
Expand Down
4 changes: 2 additions & 2 deletions .pylintrc
Original file line number Diff line number Diff line change
Expand Up @@ -52,7 +52,7 @@ confidence=
# --disable=W"
# TODO: remove R1710 (inconsistent-return-statements)
# TODO: remove C0111 (missing-docstring)
disable=similarities,fixme,I0011,E1102,R1710,C0111,C0330
disable=similarities,fixme,I0011,E1102,R1710,C0111

# Enable the message, report, category or checker with the given id(s). You can
# either give multiple identifier separated by comma (,) or put this option
Expand Down Expand Up @@ -424,4 +424,4 @@ known-third-party=enchant

# Exceptions that will emit a warning when being caught. Defaults to
# "Exception"
overgeneral-exceptions=Exception
overgeneral-exceptions=builtins.Exception
2 changes: 2 additions & 0 deletions Pipfile
Original file line number Diff line number Diff line change
Expand Up @@ -22,8 +22,10 @@ scandir = "==1.10.0"
scantree = "==0.0.1"
shapely = "==2.0.1"
tensorflow = "==2.10.0"
tensorflow-macos = {version = "==2.10.0", sys_platform = "== 'darwin'", platform_machine = "== 'arm64'"}
torch = "==1.12.1"
torchvision = "==0.13.1"
ultralytics = "==8.0.133"
werkzeug = "==2.0.0"

[dev-packages]
Expand Down
828 changes: 724 additions & 104 deletions Pipfile.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions build-electron.sh
Original file line number Diff line number Diff line change
Expand Up @@ -10,9 +10,9 @@ trap 'last_command=$current_command; current_command=$BASH_COMMAND' DEBUG
trap 'echo "Exiting shell: \"${last_command}\" command failed with exit code $?."' EXIT

echo "Running build-electron bash job!"
chmod +x setup-virtualenv.sh build-python.sh
chmod +x setup-virtualenv.sh build-python.sh
echo "Calling setup-virtualenv.sh to setup environment in portal_build"
. setup-virtualenv.sh
. setup-virtualenv.sh
echo "Calling build-python.sh to build backend executable"
. build-python.sh

Expand Down
37 changes: 25 additions & 12 deletions build-python.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,27 +11,40 @@ echo "Running build-python bash job!"
cd ./portal_build
echo "$OSTYPE"
if [[ "$OSTYPE" == "msys" ]]; then
echo "Activating .venv/Scripts/activate"
. .venv/Scripts/activate
echo "Activating .venv/Scripts/activate"
. .venv/Scripts/activate
else
echo "Activating .venv/bin/activate"
. .venv/bin/activate
echo "Activating .venv/bin/activate"
. .venv/bin/activate
fi
echo "Installing Pyinstaller..."
pip install --upgrade pyinstaller


echo "Creating flask executable..."
if [[ "$OSTYPE" == "msys" ]]; then
pyinstaller -F run.py --hidden-import pydicom.encoders.gdcm --hidden-import pydicom.encoders.pylibjpeg --hidden-import engineio.async_drivers.threading --distpath ./dist
chmod u+x ./dist/run.exe
pyinstaller \
--collect-data ultralytics \
--hidden-import pydicom.encoders.gdcm \
--hidden-import pydicom.encoders.pylibjpeg \
--hidden-import engineio.async_drivers.threading \
-F run.py \
--distpath ./dist
chmod u+x ./dist/run.exe
else
DYLD_LIBRARY_PATH=".venv/bin" pyinstaller -F run.py --hidden-import datature-hub --hidden-import engineio.async_drivers.threading --distpath ./dist
chmod u+x ./dist/run
if [ -f ./dist/run ]; then
echo "Renaming file"
mv ./dist/run ./dist/run.exe
fi
DYLD_LIBRARY_PATH=".venv/bin"
pyinstaller \
--collect-data ultralytics \
--hidden-import pydicom.encoders.gdcm \
--hidden-import pydicom.encoders.pylibjpeg \
--hidden-import engineio.async_drivers.threading \
-F run.py \
--distpath ./dist
chmod u+x ./dist/run
if [ -f ./dist/run ]; then
echo "Renaming file"
mv ./dist/run ./dist/run.exe
fi
fi

echo "Removing extra files - run.spec and build"
Expand Down
25 changes: 25 additions & 0 deletions build-windows.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,25 @@
#! /bin/bash

chmod u+x setup-virtualenv.sh
./setup-virtualenv.sh

. ./portal_build/.venv/Scripts/activate

cd src/app/
nvm install 16 && nvm use 16
npm install --legacy-peer-deps
npm run build:static

cd ../engine/
pyinstaller \
--collect-data ultralytics \
--hidden-import pydicom.encoders.gdcm \
--hidden-import pydicom.encoders.pylibjpeg \
--hidden-import engineio.async_drivers.threading \
-F run.py \
--distpath ../../portal_build/dist

chmod u+x ../../portal_build/dist/run.exe

cd ../..
npm run dist
58 changes: 57 additions & 1 deletion docs/contribution.md
Original file line number Diff line number Diff line change
Expand Up @@ -8,19 +8,75 @@ This file serves as a guide for you to contribute your ideas onto the Portal pla

[What are the Components of Portal?](#components)

[How do I Set Up Portal for Development?](#setting-up)

[How do I Get Started on Contributing?](#contributing)

[How do I Handle Errors while Contributing?](#errors)

[I'm Facing an Issue!](./issues.md)

## What are the Components of Portal? <a name="components"></a>

The two main components of portal is the **App** and the **Engine**.

The **App** serves as a graphical user interface for users to interact with.
Events triggerred in **App** may update **App** itself and/or trigger API calls to the **Engine**.
Events triggered in **App** may update **App** itself and/or trigger API calls to the **Engine**.

The **Engine** performs the more complicated computations. The **Engine** receives a specific API call from the **App**, and performs the function corresponding to the API call, before responding to **App**.

## How do I Set Up Portal for Development? <a name="setting-up"></a>

### <ins>Web Application</ins>

#### Prerequisites

- **Node v16**. You may use [nvm](https://github.com/nvm-sh/nvm) to manage your Node versions.
- **Python 3.7**. You may run the following commands to install a virtual environment with all the necessary dependencies.

```bash
chmod u+x setup-virtualenv.sh
./setup-virtualenv.sh
source ./portal_build/.venv/Scripts/activate
```

Alternatively, you can consider other virtual environment managers such as [virtualenvwrapper](https://virtualenvwrapper.readthedocs.io/en/latest/) or [conda](https://docs.conda.io/en/latest/miniconda.html), follow the installation and environment creation instructions, and then run the following command to install the dependencies.

```bash
pip install -r requirements.txt
```

#### Initializing Portal

Open two terminals. In the first terminal, navigate to `src/app` from the root directory of the repository and run the following commands:

```bash
nvm install 16 && nvm use 16
npm install --legacy-peer-deps
npm run build:static

npm run dev
```

In the second terminal, navigate to `src/engine` from the root directory of the repository and run the following commands:

```bash
python run.py
```

Lastly, open your browser and navigate to `localhost:3000` to access Portal. You should be able to see Portal's interface.

### <ins>Windows Build</ins>

We have conveniently provided a bash script to build the Windows executable for testing.

```bash
chmod u+x build-windows.sh
./build-windows.sh
```

To install the executable locally, navigate to `application` from the root directory of the repository and double click on `Portal Setup x.x.x.exe` to install Portal. Follow the instructions on the installer to complete the installation, then check the `Run Portal` box to launch Portal after installation. You should be able to see Portal's interface.

## How do I Get Started on Contributing? <a name="contributing"></a>

To begin contributing to Portal, first fork the repository, make your changes, then submit a pull request into the Portal's `develop` branch.
Expand Down
34 changes: 34 additions & 0 deletions docs/issues.md
Original file line number Diff line number Diff line change
@@ -1 +1,35 @@
# Common Issues

## General

- **Portal directory not found**: Build/cache files are stored in the following directories based on your operating system:

- **Windows EXE**:
- Absolute path: `C:\Users\$USER\AppData\Local\Programs\Portal\resources\server`
- Relative path: `resources\server`
- **Linux Build (localhost)**:
- Absolute path: `PATH/TO/portal/portal_build/server`
- Relative path: `portal_build/server`
- **Development Build**:
- Absolute path: `PATH/TO/portal/server`
- Relative path: `server`

## Windows Executable

- **No module named `{package}`**: You need to install the module. You can do this by running `pip install {package}` in the command line. If you are using a virtual environment, make sure you are in the correct environment. If you have already installed the module, make sure that you add this module as a `hidden-import` when you are building the executable using PyInstaller. You can do this by adding the `--hidden-import {package}` flag in `build-windows.sh`.

```bash
pyinstaller \
-F run.py \
--hidden-import {package} \
--distpath ../../portal_build/dist
```

- **Invalid path/File not found error (`C:\Users\$USER\AppData\Local\Temp\_MEIXXXXX\{package}\{subpackage}\{file}`)**: You need to add the package as a `datas` parameter for PyInstaller. You can do this by adding the `--collect-data` flag in `build-windows.sh`.

```bash
pyinstaller \
-F run.py \
--collect-data {package} \
--distpath ../../portal_build/dist
```
2 changes: 1 addition & 1 deletion lint.sh
100644 → 100755
Original file line number Diff line number Diff line change
@@ -1,3 +1,3 @@
#!/bin/sh

PYTHONPATH=$(readlink -f src/) pylint --rcfile=.pylintrc src/engine/*.py
PYTHONPATH=$(readlink -f src/) pylint --rcfile=.pylintrc src/engine/*.py
4 changes: 2 additions & 2 deletions package-lock.json

Some generated files are not rendered by default. Learn more about how customized files appear on GitHub.

2 changes: 1 addition & 1 deletion package.json
Original file line number Diff line number Diff line change
@@ -1,6 +1,6 @@
{
"name": "portal",
"version": "0.5.6",
"version": "0.5.8",
"author": "Datature",
"private": true,
"main": "./src/app/main.js",
Expand Down
10 changes: 5 additions & 5 deletions portal.py
Original file line number Diff line number Diff line change
Expand Up @@ -8,7 +8,7 @@

@File : portal.py
@Author : Beatrice Leong
@Version : 0.5.6
@Version : 0.5.8
@Contact : hello@datature.io
@License : Apache License 2.0
@Desc : Script to run portal from the command line.
Expand Down Expand Up @@ -43,18 +43,18 @@

if "--gpu" in arguments:
os.environ["IS_GPU"] = "0"
with open(use_gpu_dir, "w+") as gpu_flag:
with open(use_gpu_dir, "w+", encoding="utf-8") as gpu_flag:
gpu_flag.write("0")
else:
with open(use_gpu_dir, "w+") as gpu_flag:
with open(use_gpu_dir, "w+", encoding="utf-8") as gpu_flag:
gpu_flag.write("-1")

use_cache = "0"
if os.path.isfile(use_cache_dir):
with open(use_cache_dir, "r") as cache_flag:
with open(use_cache_dir, "r", encoding="utf-8") as cache_flag:
use_cache = cache_flag.read()
else:
with open(use_cache_dir, "w") as cache_flag:
with open(use_cache_dir, "w", encoding="utf-8") as cache_flag:
cache_flag.write(use_cache)

if os.getenv("IS_ELECTRON"):
Expand Down
1 change: 1 addition & 0 deletions requirements.txt
Original file line number Diff line number Diff line change
Expand Up @@ -19,4 +19,5 @@ torchvision==0.13.1
onnx==1.12.0
onnx2torch==1.5.6
onnxruntime==1.14.1
ultralytics==8.0.133
werkzeug==2.0.0
24 changes: 12 additions & 12 deletions setup-virtualenv.sh
Original file line number Diff line number Diff line change
Expand Up @@ -11,21 +11,21 @@ declare -a FILES_NEEDED=(./src/engine/run.py ./Pipfile ./Pipfile.lock)
missing_files="false"

check_if_file_exists() {
if ! test -f $1; then
missing_files="true"
echo "$1 does not exist."
fi
if ! test -f $1; then
missing_files="true"
echo "$1 does not exist."
fi
}

echo "Running setup bash job!"

for file in "${FILES_NEEDED[@]}"
do
check_if_file_exists $file
check_if_file_exists $file
done
if [[ "$missing_files" == "true" ]]; then
echo "Quitting bash job due to missing files. Ensure you have the required files before continuing."
exit 0
if [[ "$missing_files" == "true" ]]; then
echo "Quitting bash job due to missing files. Ensure you have the required files before continuing."
exit 0
fi

echo "Copying related engine files to portal_build..."
Expand All @@ -38,11 +38,11 @@ cd ./portal_build
echo "Installing Python Environment in portal_build..."
echo "$OSTYPE"
if [[ "$OSTYPE" == "msys" ]]; then
python -m pip install --upgrade pip
python -m pip install pipenv
python -m pip install --upgrade pip
python -m pip install pipenv
else
python3 -m pip install --upgrade pip
python3 -m pip install pipenv
python3 -m pip install --upgrade pip
python3 -m pip install pipenv
fi
PIPENV_VENV_IN_PROJECT=1 PIPENV_DEFAULT_PYTHON_VERSION=3.7 pipenv sync -d

Expand Down
Loading