From 2dc773546852dc91d633800b1c53fc2a78fa8345 Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 09:20:56 +0100 Subject: [PATCH 1/7] Adds github actions, a dummy tests and updates setup.py file with extra requirements --- .github/workflows/tests.yml | 29 +++++++++++++++++++++++++++++ Makefile | 6 +++++- setup.py | 5 +++++ tests/dummy_test.py | 8 ++++++++ 4 files changed, 47 insertions(+), 1 deletion(-) create mode 100644 .github/workflows/tests.yml create mode 100644 tests/dummy_test.py diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml new file mode 100644 index 0000000000..dd57e14702 --- /dev/null +++ b/.github/workflows/tests.yml @@ -0,0 +1,29 @@ +name: tests + +on: + push: + branches: [ master ] + pull_request: + branches: [ master ] + +jobs: + tests: + strategy: + matrix: + python-version: [3.6, 3.7, 3.8, 3.9] + os: ['ubuntu-latest', 'macos-latest'] + runs-on: ${{ matrix.os }} + steps: + - uses: actions/checkout@v3 + - name: Set up Python ${{ matrix.python-version }} + uses: actions/setup-python@v4 + with: + python-version: ${{ matrix.python-version }} + - name: Install dependencies + run: | + python -m pip install --upgrade pip + # cpu version of pytorch + pip install .[test] + - name: Test with pytest + run: | + make test diff --git a/Makefile b/Makefile index 301a218931..42f42c197d 100644 --- a/Makefile +++ b/Makefile @@ -14,7 +14,11 @@ docs: $(SRC) touch docs test: - nbdev_test_nbs + pytest tests + +format: + black --line-length 119 --target-version py36 tests trl examples + isort tests trl examples release: pypi nbdev_bump_version diff --git a/setup.py b/setup.py index 8467d7c990..0ed4fae480 100644 --- a/setup.py +++ b/setup.py @@ -21,6 +21,10 @@ py_versions = '2.0 2.1 2.2 2.3 2.4 2.5 2.6 2.7 3.0 3.1 3.2 3.3 3.4 3.5 3.6 3.7 3.8'.split() requirements = cfg.get('requirements','').split() +extras = { + "test" : ["pytest","pytest-xdist",], + "dev" : ["pytest","pytest-xdist", "black", "isort", "flake8>=3.8.3"], +} lic = licenses[cfg['license']] min_python = cfg['min_python'] @@ -37,6 +41,7 @@ packages = setuptools.find_packages(), include_package_data = True, install_requires = requirements, + extras_require = extras, python_requires = '>=' + cfg['min_python'], long_description = open('README.md').read(), long_description_content_type = 'text/markdown', diff --git a/tests/dummy_test.py b/tests/dummy_test.py new file mode 100644 index 0000000000..4a136bc65b --- /dev/null +++ b/tests/dummy_test.py @@ -0,0 +1,8 @@ +import pytest + + +@pytest.mark.parametrize("value", [1,2,3]) +def test_value(value): + assert value == value + + From a571298da31f71df94c9dc978d73b5606fea711b Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 09:41:43 +0100 Subject: [PATCH 2/7] removed python 36 for the moment as it is not support on ubuntu 22 --- .github/workflows/tests.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index dd57e14702..0d62fdeccc 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,7 +10,7 @@ jobs: tests: strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] + python-version: [3.7, 3.8, 3.9] os: ['ubuntu-latest', 'macos-latest'] runs-on: ${{ matrix.os }} steps: From 0096534af56230603efdd4142eac91d6f9636d60 Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 09:50:45 +0100 Subject: [PATCH 3/7] using ubuntu 20.04 as it supports python 3.6 --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 0d62fdeccc..5dcfa3b9fb 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,8 +10,8 @@ jobs: tests: strategy: matrix: - python-version: [3.7, 3.8, 3.9] - os: ['ubuntu-latest', 'macos-latest'] + python-version: [3.6, 3.7, 3.8, 3.9] + os: ['ubuntu-20.04', 'macos-latest'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3 From e9b980ebb3f3dd6859be7fd5385abb20412bbf05 Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 10:09:04 +0100 Subject: [PATCH 4/7] loosens transformer restriction --- settings.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/settings.ini b/settings.ini index dadaeb034d..cda6a8c9f5 100644 --- a/settings.ini +++ b/settings.ini @@ -14,7 +14,7 @@ language = English custom_sidebar = False license = apache2 status = 2 -requirements = torch>=1.4.0 transformers==4.3.2 numpy>=1.18.2 +requirements = torch>=1.4.0 transformers==4.25.1 numpy>=1.18.2 nbs_path = ./nbs/ doc_path = docs doc_host = https://lvwerra.github.io From 44fef31f04ebcd1806f36d589c0c861536d7d68d Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 10:09:26 +0100 Subject: [PATCH 5/7] adds test of gpt2 model trainer --- tests/dummy_test.py | 8 -------- tests/test_gpt2_model.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 35 insertions(+), 8 deletions(-) delete mode 100644 tests/dummy_test.py create mode 100644 tests/test_gpt2_model.py diff --git a/tests/dummy_test.py b/tests/dummy_test.py deleted file mode 100644 index 4a136bc65b..0000000000 --- a/tests/dummy_test.py +++ /dev/null @@ -1,8 +0,0 @@ -import pytest - - -@pytest.mark.parametrize("value", [1,2,3]) -def test_value(value): - assert value == value - - diff --git a/tests/test_gpt2_model.py b/tests/test_gpt2_model.py new file mode 100644 index 0000000000..67e5e954c7 --- /dev/null +++ b/tests/test_gpt2_model.py @@ -0,0 +1,35 @@ +# imports +import pytest +import torch +from transformers import GPT2Tokenizer +from trl.gpt2 import GPT2HeadWithValueModel, respond_to_batch +from trl.ppo import PPOTrainer + + + +def test_gpt2_model(): + # get models + gpt2_model = GPT2HeadWithValueModel.from_pretrained('gpt2') + gpt2_model_ref = GPT2HeadWithValueModel.from_pretrained('gpt2') + gpt2_tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + + # initialize trainer + ppo_config = {'batch_size': 1, 'forward_batch_size': 1} + ppo_trainer = PPOTrainer(gpt2_model, gpt2_model_ref, gpt2_tokenizer, **ppo_config) + + # encode a query + query_txt = "This morning I went to the " + query_tensor = gpt2_tokenizer.encode(query_txt, return_tensors="pt") + + # get model response + response_tensor = respond_to_batch(gpt2_model, query_tensor) + response_txt = gpt2_tokenizer.decode(response_tensor[0,:]) + + # define a reward for response + # (this could be any reward such as human feedback or output from another model) + reward = [torch.tensor(1.0)] + + # train model with ppo + train_stats = ppo_trainer.step([query_tensor[0]], [response_tensor[0]], reward) + + #print(train_stats) \ No newline at end of file From 389bb273e9d1eb0a006e834654eed950b6559a4c Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 10:20:32 +0100 Subject: [PATCH 6/7] updates test and transformers version --- settings.ini | 2 +- tests/test_gpt2_model.py | 52 +++++++++++++++++++++++++++++++++------- 2 files changed, 44 insertions(+), 10 deletions(-) diff --git a/settings.ini b/settings.ini index cda6a8c9f5..081c8e68ee 100644 --- a/settings.ini +++ b/settings.ini @@ -14,7 +14,7 @@ language = English custom_sidebar = False license = apache2 status = 2 -requirements = torch>=1.4.0 transformers==4.25.1 numpy>=1.18.2 +requirements = torch>=1.4.0 transformers>=4.18.0 numpy>=1.18.2 nbs_path = ./nbs/ doc_path = docs doc_host = https://lvwerra.github.io diff --git a/tests/test_gpt2_model.py b/tests/test_gpt2_model.py index 67e5e954c7..10bd61a4c8 100644 --- a/tests/test_gpt2_model.py +++ b/tests/test_gpt2_model.py @@ -6,24 +6,24 @@ from trl.ppo import PPOTrainer - def test_gpt2_model(): # get models - gpt2_model = GPT2HeadWithValueModel.from_pretrained('gpt2') - gpt2_model_ref = GPT2HeadWithValueModel.from_pretrained('gpt2') - gpt2_tokenizer = GPT2Tokenizer.from_pretrained('gpt2') + gpt2_model = GPT2HeadWithValueModel.from_pretrained("gpt2") + gpt2_model_ref = GPT2HeadWithValueModel.from_pretrained("gpt2") + gpt2_tokenizer = GPT2Tokenizer.from_pretrained("gpt2") # initialize trainer - ppo_config = {'batch_size': 1, 'forward_batch_size': 1} + ppo_config = {"batch_size": 1, "forward_batch_size": 1} ppo_trainer = PPOTrainer(gpt2_model, gpt2_model_ref, gpt2_tokenizer, **ppo_config) # encode a query query_txt = "This morning I went to the " query_tensor = gpt2_tokenizer.encode(query_txt, return_tensors="pt") - + assert query_tensor.shape == (1, 7) # get model response - response_tensor = respond_to_batch(gpt2_model, query_tensor) - response_txt = gpt2_tokenizer.decode(response_tensor[0,:]) + response_tensor = respond_to_batch(gpt2_model, query_tensor) + assert response_tensor.shape == (1, 20) + response_txt = gpt2_tokenizer.decode(response_tensor[0, :]) # define a reward for response # (this could be any reward such as human feedback or output from another model) @@ -32,4 +32,38 @@ def test_gpt2_model(): # train model with ppo train_stats = ppo_trainer.step([query_tensor[0]], [response_tensor[0]], reward) - #print(train_stats) \ No newline at end of file + EXPECTED_STATS = [ + "objective/kl", + "objective/kl_dist", + "objective/logprobs", + "objective/ref_logprobs", + "objective/kl_coef", + "objective/entropy", + "ppo/mean_non_score_reward", + "ppo/loss/policy", + "ppo/loss/value", + "ppo/loss/total", + "ppo/policy/entropy", + "ppo/policy/approxkl", + "ppo/policy/policykl", + "ppo/policy/clipfrac", + "ppo/policy/advantages", + "ppo/policy/advantages_mean", + "ppo/policy/ratio", + "ppo/returns/mean", + "ppo/returns/var", + "ppo/val/vpred", + "ppo/val/error", + "ppo/val/clipfrac", + "ppo/val/mean", + "ppo/val/var", + "ppo/val/var_explained", + "time/ppo/forward_pass", + "time/ppo/compute_rewards", + "time/ppo/optimize_step", + "time/ppo/calc_stats", + "time/ppo/total", + ] + + for stat in EXPECTED_STATS: + assert stat in train_stats.keys() From a69b11cf3510bc6c51aa17f2ed417e5ff5832eec Mon Sep 17 00:00:00 2001 From: Edward Beeching Date: Wed, 21 Dec 2022 10:57:05 +0100 Subject: [PATCH 7/7] removes 3.6 from test suite, adds windows to test suite --- .github/workflows/tests.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml index 5dcfa3b9fb..9534a8cee9 100644 --- a/.github/workflows/tests.yml +++ b/.github/workflows/tests.yml @@ -10,8 +10,8 @@ jobs: tests: strategy: matrix: - python-version: [3.6, 3.7, 3.8, 3.9] - os: ['ubuntu-20.04', 'macos-latest'] + python-version: [3.7, 3.8, 3.9] + os: ['ubuntu-latest', 'macos-latest', 'windows-latest'] runs-on: ${{ matrix.os }} steps: - uses: actions/checkout@v3