diff --git a/.github/FUNDING.yml b/.github/FUNDING.yml
new file mode 100644
index 0000000000..e08de2c074
--- /dev/null
+++ b/.github/FUNDING.yml
@@ -0,0 +1,13 @@
+# These are supported funding model platforms
+
+github: [piskvorky] # Replace with up to 4 GitHub Sponsors-enabled usernames e.g., [user1, user2]
+patreon: # Replace with a single Patreon username
+open_collective: # Replace with a single Open Collective username
+ko_fi: # Replace with a single Ko-fi username
+tidelift: # Replace with a single Tidelift platform-name/package-name e.g., npm/babel
+community_bridge: # Replace with a single Community Bridge project-name e.g., cloud-foundry
+liberapay: # Replace with a single Liberapay username
+issuehunt: # Replace with a single IssueHunt username
+otechie: # Replace with a single Otechie username
+custom: # Replace with up to 4 custom sponsorship URLs e.g., ['link1', 'link2']
+
diff --git a/.github/workflows/build-wheels.yml b/.github/workflows/build-wheels.yml
new file mode 100644
index 0000000000..5430dcdb7a
--- /dev/null
+++ b/.github/workflows/build-wheels.yml
@@ -0,0 +1,86 @@
+name: Build wheels
+
+on:
+ push:
+ branches: [ develop ]
+ pull_request:
+ branches: [ develop ]
+ schedule:
+ - cron: '0 0 * * sun,wed'
+
+jobs:
+ build:
+ runs-on: ${{ matrix.os }}
+ defaults:
+ run:
+ shell: bash
+ strategy:
+ fail-fast: false
+ matrix:
+ python-version: [3.6, 3.7, 3.8]
+ os: [ubuntu-latest, macos-latest]
+ platform: [x64]
+ include:
+ - os: ubuntu-latest
+ python-version: 3.7
+ skip-network-tests: 1
+ - os: ubuntu-latest
+ python-version: 3.8
+ skip-network-tests: 1
+ - os: macos-latest
+ travis-os-name: osx # For multibuild
+ skip-network-tests: 1
+ env:
+ PKG_NAME: gensim
+ REPO_DIR: gensim
+ BUILD_COMMIT: HEAD
+ PLAT: x86_64
+ UNICODE_WIDTH: 32
+ MB_PYTHON_VERSION: ${{ matrix.python-version }} # MB_PYTHON_VERSION is needed by Multibuild
+ TEST_DEPENDS: Morfessor==2.0.2a4 python-levenshtein==0.12.0 visdom==0.1.8.9 pytest mock cython nmslib pyemd testfixtures scikit-learn pyemd
+ DOCKER_TEST_IMAGE: multibuild/xenial_x86_64
+ TRAVIS_OS_NAME: ${{ matrix.travis-os-name }}
+ SKIP_NETWORK_TESTS: ${{ matrix.skip-network-tests }}
+
+ steps:
+ - uses: actions/checkout@v2
+ with:
+ submodules: recursive
+ fetch-depth: 0
+ - name: Print environment variables
+ run: |
+ echo "PLAT: ${PLAT}"
+ echo "DOCKER_TEST_IMAGE: ${DOCKER_TEST_IMAGE}"
+ echo "TEST_DEPENDS: ${TEST_DEPENDS}"
+ echo "TRAVIS_OS_NAME: ${TRAVIS_OS_NAME}"
+ echo "SKIP_NETWORK_TESTS: ${SKIP_NETWORK_TESTS}"
+ - name: Set up Python ${{ matrix.python-version }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python-version }}
+ - name: Install dependencies
+ run: |
+ python -m pip install --upgrade pip
+ pip install virtualenv
+ - name: Build and Install Wheels
+ run: |
+ echo ::group::Set up Multibuild
+ source multibuild/common_utils.sh
+ source multibuild/travis_steps.sh
+ source config.sh
+ echo ::endgroup::
+ echo ::group::Before install
+ before_install
+ echo ::endgroup::
+ echo ::group::Build wheel
+ build_wheel $REPO_DIR ${{ matrix.PLAT }}
+ echo ::endgroup::
+ echo ::group::Install run
+ install_run ${{ matrix.PLAT }}
+ echo ::endgroup::
+ - name: Upload wheels to s3://gensim-wheels
+ if: always()
+ run: |
+ pip install wheelhouse-uploader
+ ls wheelhouse/*.whl
+ python -m wheelhouse_uploader upload --local-folder wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn
diff --git a/.github/workflows/tests.yml b/.github/workflows/tests.yml
new file mode 100644
index 0000000000..8e3ad48871
--- /dev/null
+++ b/.github/workflows/tests.yml
@@ -0,0 +1,60 @@
+name: Tests
+on:
+ push:
+ branches: [ develop ]
+ pull_request:
+ branches: [ develop ]
+
+jobs:
+ tests:
+ name: ${{ matrix.name }}
+ runs-on: ${{ matrix.os }}
+ defaults:
+ run:
+ shell: bash
+ strategy:
+ fail-fast: false
+ matrix:
+ include:
+ - {name: Linux, python: 3.6, os: ubuntu-20.04, tox: 'flake8,flake8-docs'}
+ - {name: Linux, python: 3.6, os: ubuntu-20.04, tox: 'py36-linux'}
+ - {name: Linux, python: 3.7, os: ubuntu-20.04, tox: 'py37-linux'}
+ - {name: Linux, python: 3.8, os: ubuntu-20.04, tox: 'py38-linux'}
+ env:
+ TOX_PARALLEL_NO_SPINNER: 1
+
+ steps:
+ - uses: actions/checkout@v2
+ - name: Setup up Python ${{ matrix.python }}
+ uses: actions/setup-python@v2
+ with:
+ python-version: ${{ matrix.python }}
+ - name: Update pip
+ run: python -m pip install -U pip
+
+ #
+ # Work-around mysterious build problem
+ # https://github.com/RaRe-Technologies/gensim/pull/3078/checks?check_run_id=2117914443
+ # https://www.scala-sbt.org/1.x/docs/Installing-sbt-on-Linux.html
+ #
+ - name: Update sbt
+ run: |
+ echo "deb https://dl.bintray.com/sbt/debian /" | sudo tee -a /etc/apt/sources.list.d/sbt.list
+ curl -sL "https://keyserver.ubuntu.com/pks/lookup?op=get&search=0x2EE0EA64E40A89B84B2DF73499E82A75642AC823" | sudo apt-key add
+ sudo apt-get update -y
+ sudo apt-get install -y sbt
+ - name: Install tox, gdb
+ run: |
+ pip install tox
+ sudo apt-get update -y
+ sudo apt-get install -y gdb
+ - name: Enable core dumps
+ run: ulimit -c unlimited -S # enable core dumps
+ - name: Run tox tests
+ run: tox -e ${{ matrix.tox }}
+ - name: Collect corefile
+ if: ${{ failure() }}
+ run: |
+ pwd
+ COREFILE=$(find . -maxdepth 1 -name "core*" | head -n 1)
+ if [[ -f "$COREFILE" ]]; then EXECFILE=$(gdb -c "$COREFILE" -batch | grep "Core was generated" | tr -d "\`" | cut -d' ' -f5); file "$COREFILE"; gdb -c "$COREFILE" "$EXECFILE" -x continuous_integration/debug.gdb -batch; fi
diff --git a/.gitignore b/.gitignore
index c3ba120f37..019e1812f7 100644
--- a/.gitignore
+++ b/.gitignore
@@ -76,3 +76,18 @@ data
*.inv
*.js
docs/_images/
+
+#
+# Generated by Cython
+#
+gensim/_matutils.c
+gensim/corpora/_mmreader.c
+gensim/models/doc2vec_corpusfile.cpp
+gensim/models/doc2vec_inner.cpp
+gensim/models/fasttext_corpusfile.cpp
+gensim/models/fasttext_inner.c
+gensim/models/nmf_pgd.c
+gensim/models/word2vec_corpusfile.cpp
+gensim/models/word2vec_inner.c
+
+.ipynb_checkpoints
diff --git a/.gitmodules b/.gitmodules
new file mode 100644
index 0000000000..347fe93043
--- /dev/null
+++ b/.gitmodules
@@ -0,0 +1,3 @@
+[submodule "multibuild"]
+ path = multibuild
+ url = https://github.com/matthew-brett/multibuild.git
diff --git a/.travis.yml b/.travis.yml
index 553a8215d8..b887cac1b6 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -1,53 +1,52 @@
-sudo: false
-
-cache:
- apt: true
- directories:
- - $HOME/.cache/pip
- - $HOME/.ccache
- - $HOME/.pip-cache
-dist: trusty
+branches:
+ only:
+ - /v\d+\.\d+\.\d+/
language: python
+arch: arm64-graviton2
+dist: focal
+virt: vm
+group: edge
+services: docker
env:
- TOX_PARALLEL_NO_SPINNER: 1
-
+ global:
+ - REPO_DIR=gensim
+ - BUILD_COMMIT=HEAD
+ - UNICODE_WIDTH=32
+ - PLAT=aarch64
+ - MB_ML_VER=2014
+ - SKIP_NETWORK_TESTS=1
+ - DOCKER_TEST_IMAGE=multibuild/xenial_arm64v8
+ - BUILD_DEPENDS="numpy==1.19.2 scipy==1.5.3"
+ - TEST_DEPENDS="pytest mock cython nmslib pyemd testfixtures Morfessor==2.0.2a4 python-levenshtein==0.12.0 visdom==0.1.8.9 scikit-learn"
matrix:
- include:
- - python: '3.6'
- env: TOXENV="flake8,flake8-docs"
-
- - python: '3.8'
+ - os: linux
env:
- - TOXENV="py38-linux"
- dist: bionic
-
- - python: '3.7'
+ - MB_PYTHON_VERSION=3.6
+ - os: linux
env:
- - TOXENV="py37-linux"
- # The following two lines used to be necessary because Travis left files lying around in ~/.aws/,
- # messing up our tests. Now fixed since https://github.com/travis-ci/travis-ci/issues/7940
- # - BOTO_CONFIG="/dev/null"
- #sudo: true
- dist: xenial
-
- - python: '3.6'
- env: TOXENV="py36-linux"
-
-
+ - MB_PYTHON_VERSION=3.7
+ - os: linux
+ env:
+ - MB_PYTHON_VERSION=3.8
+ - os: linux
+ env:
+ - MB_PYTHON_VERSION=3.9
+before_install:
+ - source multibuild/common_utils.sh
+ - source multibuild/travis_steps.sh
+ - before_install
install:
- - pip install tox
- - sudo apt-get install -y gdb
-
-
-before_script:
- - ulimit -c unlimited -S # enable core dumps
-
-
-script: tox -vv
-
-
-after_failure:
- - pwd
- - COREFILE=$(find . -maxdepth 1 -name "core*" | head -n 1)
- - if [[ -f "$COREFILE" ]]; then EXECFILE=$(gdb -c "$COREFILE" -batch | grep "Core was generated" | tr -d "\`" | cut -d' ' -f5); file "$COREFILE"; gdb -c "$COREFILE" "$EXECFILE" -x continuous_integration/debug.gdb -batch; fi
+ - build_wheel $REPO_DIR $PLAT
+script:
+ - install_run $PLAT
+after_script:
+ - ls -laht ${TRAVIS_BUILD_DIR}/wheelhouse/
+ - pip install wheelhouse-uploader
+ - python -m wheelhouse_uploader upload --local-folder ${TRAVIS_BUILD_DIR}/wheelhouse/ --no-ssl-check gensim-wheels --provider S3 --no-enable-cdn
+
+notifications:
+ email:
+ - penkov+gensimwheels@pm.me
+ on_success: always
+ on_failure: always
diff --git a/CHANGELOG.md b/CHANGELOG.md
index 7108bd1516..d64c1076e1 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -1,6 +1,87 @@
Changes
=======
+## 4.0.0.rc1, 2021-03-19
+
+**⚠️ Gensim 4.0 contains breaking API changes! See the [Migration guide](https://github.com/RaRe-Technologies/gensim/wiki/Migrating-from-Gensim-3.x-to-4) to update your existing Gensim 3.x code and models.**
+
+Gensim 4.0 is a major release with lots of performance & robustness improvements and a new website.
+
+### Main highlights (see also *👍 Improvements* below)
+
+* Massively optimized popular algorithms the community has grown to love: [fastText](https://radimrehurek.com/gensim/models/fasttext.html), [word2vec](https://radimrehurek.com/gensim/models/word2vec.html), [doc2vec](https://radimrehurek.com/gensim/models/doc2vec.html), [phrases](https://radimrehurek.com/gensim/models/phrases.html):
+
+ a. **Efficiency**
+
+ | model | 3.8.3: wall time / peak RAM / throughput | 4.0.0: wall time / peak RAM / throughput |
+ |----------|------------|--------|
+ | fastText | 2.9h / 4.11 GB / 822k words/s | 2.3h / **1.26 GB** / 914k words/s |
+ | word2vec | 1.7h / 0.36 GB / 1685k words/s | **1.2h** / 0.33 GB / 1762k words/s |
+
+ In other words, fastText now needs 3x less RAM (and is faster); word2vec has 2x faster init (and needs less RAM, and is faster); detecting collocation phrases is 2x faster. ([4.0 benchmarks](https://github.com/RaRe-Technologies/gensim/issues/2887#issuecomment-711097334))
+
+ b. **Robustness**. We fixed a bunch of long-standing bugs by refactoring the internal code structure (see 🔴 Bug fixes below)
+
+ c. **Simplified OOP model** for easier model exports and integration with TensorFlow, PyTorch &co.
+
+ These improvements come to you transparently aka "for free", but see [Migration guide](https://github.com/RaRe-Technologies/gensim/wiki/Migrating-from-Gensim-3.x-to-4) for some changes that break the old Gensim 3.x API. **Update your code accordingly**.
+
+* Dropped a bunch of externally contributed modules: summarization, pivoted TFIDF normalization, FIXME.
+ - Code quality was not up to our standards. Also there was no one to maintain them, answer user questions, support these modules.
+
+ So rather than let them rot, we took the hard decision of removing these contributed modules from Gensim. If anyone's interested in maintaining them please fork into your own repo, they can live happily outside of Gensim.
+
+* Dropped Python 2. Gensim 4.0 is Py3.6+. Read our [Python version support policy](https://github.com/RaRe-Technologies/gensim/wiki/Gensim-And-Compatibility).
+ - If you still need Python 2 for some reason, stay at [Gensim 3.8.3](https://github.com/RaRe-Technologies/gensim/releases/tag/3.8.3).
+
+* A new [Gensim website](https://radimrehurek.com/gensim_4.0.0) – finally! 🙃
+
+So, a major clean-up release overall. We're happy with this **tighter, leaner and faster Gensim**.
+
+This is the direction we'll keep going forward: less kitchen-sink of "latest academic algorithms", more focus on robust engineering, targetting common concrete NLP & document similarity use-cases.
+
+### :star2: New Features
+
+* Default to pickle protocol 4 when saving models (__[piskvorky](https://github.com/piskvorky)__, [#3065](https://github.com/RaRe-Technologies/gensim/pull/3065))
+* Record lifecycle events in Gensim models (__[piskvorky](https://github.com/piskvorky)__, [#3060](https://github.com/RaRe-Technologies/gensim/pull/3060))
+* Make WMD normalization optional (__[piskvorky](https://github.com/piskvorky)__, [#3073](https://github.com/RaRe-Technologies/gensim/pull/3073))
+
+### :red_circle: Bug fixes
+
+* fix RuntimeError in export_phrases (change defaultdict to dict) (__[thalishsajeed](https://github.com/thalishsajeed)__, [#3041](https://github.com/RaRe-Technologies/gensim/pull/3041))
+
+### :books: Tutorial and doc improvements
+
+* fix various documentation warnings (__[mpenkov](https://github.com/mpenkov)__, [#3077](https://github.com/RaRe-Technologies/gensim/pull/3077))
+* Fix broken link in run_doc how-to (__[sezanzeb](https://github.com/sezanzeb)__, [#2991](https://github.com/RaRe-Technologies/gensim/pull/2991))
+* Point WordEmbeddingSimilarityIndex documentation to gensim.similarities (__[Witiko](https://github.com/Witiko)__, [#3003](https://github.com/RaRe-Technologies/gensim/pull/3003))
+* Make the link to the Gensim 3.8.3 documentation dynamic (__[Witiko](https://github.com/Witiko)__, [#2996](https://github.com/RaRe-Technologies/gensim/pull/2996))
+
+### :+1: Improvements
+
+### :warning: Removed functionality
+
+* remove on_batch_begin and on_batch_end callbacks (__[mpenkov](https://github.com/mpenkov)__, [#3078](https://github.com/RaRe-Technologies/gensim/pull/3078))
+* remove pattern dependency (__[mpenkov](https://github.com/mpenkov)__, [#3012](https://github.com/RaRe-Technologies/gensim/pull/3012))
+* rm gensim.viz submodule (__[mpenkov](https://github.com/mpenkov)__, [#3055](https://github.com/RaRe-Technologies/gensim/pull/3055))
+
+### :warning: Deprecations (will be removed in the next major release)
+
+### ??? Misc
+
+**FIXME** This is a list of PRs that I couldn't find an appropriate section for.
+We could make some other section for them or remove them from the changelog entirely.
+This is probably OK as-is for the release candidate, but we should clean this up for the proper, final release.
+
+* [MRG] Add Github sponsor + donation nags (__[piskvorky](https://github.com/piskvorky)__, [#3069](https://github.com/RaRe-Technologies/gensim/pull/3069))
+* Update URLs (__[jonaschn](https://github.com/jonaschn)__, [#3063](https://github.com/RaRe-Technologies/gensim/pull/3063))
+* Fix race condition in FastText tests (__[sleepy-owl](https://github.com/sleepy-owl)__, [#3059](https://github.com/RaRe-Technologies/gensim/pull/3059))
+* Add py39 wheels to travis/azure (__[FredHappyface](https://github.com/FredHappyface)__, [#3058](https://github.com/RaRe-Technologies/gensim/pull/3058))
+* Update repos before trying to install gdb (__[janaknat](https://github.com/janaknat)__, [#3035](https://github.com/RaRe-Technologies/gensim/pull/3035))
+* transformed camelCase to snake_case test names (__[sezanzeb](https://github.com/sezanzeb)__, [#3033](https://github.com/RaRe-Technologies/gensim/pull/3033))
+* move x86 tests from Travis to GHA, add aarch64 wheel build to Travis (__[janaknat](https://github.com/janaknat)__, [#3026](https://github.com/RaRe-Technologies/gensim/pull/3026))
+* Add Github Actions x86 and mac jobs to build python wheels (__[janaknat](https://github.com/janaknat)__, [#3024](https://github.com/RaRe-Technologies/gensim/pull/3024))
+
## 4.0.0beta, 2020-10-31
**⚠️ Gensim 4.0 contains breaking API changes! See the [Migration guide](https://github.com/RaRe-Technologies/gensim/wiki/Migrating-from-Gensim-3.x-to-4) to update your existing Gensim 3.x code and models.**
@@ -104,6 +185,7 @@ Production stability is important to Gensim, so we're improving the process of *
* [#2926](https://github.com/RaRe-Technologies/gensim/pull/2926): Rename `num_words` to `topn` in dtm_coherence, by [@MeganStodel](https://github.com/MeganStodel)
* [#2937](https://github.com/RaRe-Technologies/gensim/pull/2937): Remove Keras dependency, by [@piskvorky](https://github.com/piskvorky)
* Removed all code, methods, attributes and functions marked as deprecated in [Gensim 3.8.3](https://github.com/RaRe-Technologies/gensim/releases/tag/3.8.3).
+* Removed pattern dependency (PR [#3012](https://github.com/RaRe-Technologies/gensim/pull/3012), [@mpenkov](https://github.com/mpenkov)). If you need to lemmatize, do it prior to passing the corpus to gensim.
---
diff --git a/ISSUE_TEMPLATE.md b/ISSUE_TEMPLATE.md
index ec31c5964f..95121b30ae 100644
--- a/ISSUE_TEMPLATE.md
+++ b/ISSUE_TEMPLATE.md
@@ -15,6 +15,12 @@ What are you trying to achieve? What is the expected result? What are you seeing
Include full tracebacks, logs and datasets if necessary. Please keep the examples minimal ("minimal reproducible example").
+If your problem is with a specific Gensim model (word2vec, lsimodel, doc2vec, fasttext, ldamodel etc), include the following:
+
+```python
+print(my_model.lifecycle_events)
+```
+
#### Versions
Please provide the output of:
diff --git a/azure-pipelines.yml b/azure-pipelines.yml
index c624e7e3e9..8e8102fa12 100644
--- a/azure-pipelines.yml
+++ b/azure-pipelines.yml
@@ -12,6 +12,9 @@ strategy:
py38:
python.version: '3.8'
TOXENV: "py38-win"
+ py39:
+ python.version: '3.9'
+ TOXENV: "py39-win"
steps:
- task: UsePythonVersion@0
@@ -25,5 +28,5 @@ steps:
displayName: 'Install tox'
- script: |
- tox -vv
+ tox
displayName: 'Testing'
diff --git a/config.sh b/config.sh
new file mode 100755
index 0000000000..8a2be6e0dd
--- /dev/null
+++ b/config.sh
@@ -0,0 +1,27 @@
+# Define custom utilities
+
+function pre_build {
+ # Any stuff that you need to do before you start building the wheels
+ # Runs in the root directory of this repository.
+ :
+}
+
+function build_wheel_cmd {
+ local cmd=${1:-pip_wheel_cmd}
+ local wheelhouse=$(abspath ${WHEEL_SDIR:-wheelhouse})
+ start_spinner
+ if [ -n "$(is_function "pre_build")" ]; then pre_build; fi
+ stop_spinner
+ if [ -n "$BUILD_DEPENDS" ]; then
+ pip install $(pip_opts) $BUILD_DEPENDS
+ fi
+ $cmd $wheelhouse
+ repair_wheelhouse $wheelhouse
+}
+
+function run_tests {
+ # Runs tests on installed distribution from an empty directory
+ python --version
+ pip freeze
+ pytest -rfxEXs --durations=20 --disable-warnings --showlocals --pyargs gensim
+}
diff --git a/docs/notebooks/nmf_tutorial.ipynb b/docs/notebooks/nmf_tutorial.ipynb
index a1fa18c00a..d7eb67b7eb 100644
--- a/docs/notebooks/nmf_tutorial.ipynb
+++ b/docs/notebooks/nmf_tutorial.ipynb
@@ -139,7 +139,7 @@
"import gensim.downloader\n",
"from gensim import matutils, utils\n",
"from gensim.corpora import Dictionary\n",
- "from gensim.models import CoherenceModel, LdaModel, TfidfModel\n",
+ "from gensim.models import CoherenceModel, LdaModel, TfidfModel, LsiModel\n",
"from gensim.models.basemodel import BaseTopicModel\n",
"from gensim.models.nmf import Nmf as GensimNmf\n",
"from gensim.parsing.preprocessing import preprocess_string\n",
@@ -191,16 +191,7 @@
"cell_type": "code",
"execution_count": 3,
"metadata": {},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2019-06-17 00:24:02,672 : WARNING : this function is deprecated, use smart_open.open instead\n",
- "2019-06-17 00:24:03,509 : WARNING : this function is deprecated, use smart_open.open instead\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"random_state = RandomState(42)\n",
"\n",
@@ -265,11 +256,11 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "2019-06-17 00:24:12,357 : INFO : adding document #0 to Dictionary(0 unique tokens: [])\n",
- "2019-06-17 00:24:12,761 : INFO : built Dictionary(25279 unique tokens: ['actual', 'assum', 'babbl', 'batka', 'batkaj']...) from 2819 documents (total 435328 corpus positions)\n",
- "2019-06-17 00:24:12,803 : INFO : discarding 18198 tokens: [('batka', 1), ('batkaj', 1), ('beatl', 1), ('ccmail', 3), ('dayton', 4), ('edu', 1785), ('inhibit', 1), ('jbatka', 1), ('line', 2748), ('organ', 2602)]...\n",
- "2019-06-17 00:24:12,804 : INFO : keeping 7081 tokens which were in no less than 5 and no more than 1409 (=50.0%) documents\n",
- "2019-06-17 00:24:12,815 : INFO : resulting dictionary: Dictionary(7081 unique tokens: ['actual', 'assum', 'babbl', 'burster', 'caus']...)\n"
+ "2019-05-06 15:57:16,471 : INFO : adding document #0 to Dictionary(0 unique tokens: [])\n",
+ "2019-05-06 15:57:16,781 : INFO : built Dictionary(25279 unique tokens: ['sketch', 'addario', 'foyer', 'labratsat', 'reclaim']...) from 2819 documents (total 435328 corpus positions)\n",
+ "2019-05-06 15:57:16,809 : INFO : discarding 18198 tokens: [('batka', 1), ('batkaj', 1), ('beatl', 1), ('ccmail', 3), ('dayton', 4), ('edu', 1785), ('inhibit', 1), ('jbatka', 1), ('line', 2748), ('organ', 2602)]...\n",
+ "2019-05-06 15:57:16,810 : INFO : keeping 7081 tokens which were in no less than 5 and no more than 1409 (=50.0%) documents\n",
+ "2019-05-06 15:57:16,821 : INFO : resulting dictionary: Dictionary(7081 unique tokens: ['colost', 'choke', 'editor', 'china', 'piss']...)\n"
]
}
],
@@ -379,75 +370,75 @@
"name": "stderr",
"output_type": "stream",
"text": [
- "2019-06-17 00:24:15,115 : INFO : running NMF training, 5 topics, 5 passes over the supplied corpus of 2819 documents, evaluating l2 norm every 2819 documents\n",
- "2019-06-17 00:24:15,136 : INFO : PROGRESS: pass 0, at document #1000/2819\n",
- "2019-06-17 00:24:15,162 : INFO : W error diff: -inf\n",
- "2019-06-17 00:24:15,182 : INFO : PROGRESS: pass 0, at document #2000/2819\n",
- "2019-06-17 00:24:15,202 : INFO : W error diff: -2.1783593508632997\n",
- "2019-06-17 00:24:15,221 : INFO : PROGRESS: pass 0, at document #2819/2819\n",
- "2019-06-17 00:24:15,606 : INFO : L2 norm: 28.137070033533682\n",
- "2019-06-17 00:24:15,658 : INFO : topic #0 (0.404): 0.011*\"isra\" + 0.010*\"israel\" + 0.007*\"arab\" + 0.006*\"jew\" + 0.005*\"palestinian\" + 0.004*\"henri\" + 0.003*\"toronto\" + 0.003*\"question\" + 0.003*\"kill\" + 0.003*\"polici\"\n",
- "2019-06-17 00:24:15,660 : INFO : topic #1 (0.358): 0.009*\"space\" + 0.005*\"access\" + 0.005*\"nasa\" + 0.004*\"pat\" + 0.003*\"digex\" + 0.003*\"orbit\" + 0.003*\"shuttl\" + 0.003*\"graphic\" + 0.003*\"data\" + 0.003*\"com\"\n",
- "2019-06-17 00:24:15,661 : INFO : topic #2 (0.388): 0.013*\"armenian\" + 0.006*\"turkish\" + 0.005*\"greek\" + 0.005*\"peopl\" + 0.004*\"armenia\" + 0.004*\"turk\" + 0.004*\"argic\" + 0.004*\"bike\" + 0.003*\"serdar\" + 0.003*\"turkei\"\n",
- "2019-06-17 00:24:15,663 : INFO : topic #3 (0.423): 0.010*\"moral\" + 0.006*\"keith\" + 0.004*\"anim\" + 0.004*\"jake\" + 0.003*\"boni\" + 0.003*\"act\" + 0.003*\"instinct\" + 0.003*\"think\" + 0.003*\"caltech\" + 0.003*\"object\"\n",
- "2019-06-17 00:24:15,665 : INFO : topic #4 (0.441): 0.009*\"islam\" + 0.009*\"god\" + 0.006*\"muslim\" + 0.006*\"livesei\" + 0.005*\"imag\" + 0.005*\"sgi\" + 0.005*\"jaeger\" + 0.004*\"jon\" + 0.004*\"solntz\" + 0.004*\"wpd\"\n",
- "2019-06-17 00:24:15,679 : INFO : W error diff: -0.6087333117616911\n",
- "2019-06-17 00:24:15,718 : INFO : PROGRESS: pass 1, at document #1000/2819\n",
- "2019-06-17 00:24:15,735 : INFO : W error diff: -1.5858439279007879\n",
- "2019-06-17 00:24:15,762 : INFO : PROGRESS: pass 1, at document #2000/2819\n",
- "2019-06-17 00:24:15,773 : INFO : W error diff: -1.1329837530094071\n",
- "2019-06-17 00:24:15,791 : INFO : PROGRESS: pass 1, at document #2819/2819\n",
- "2019-06-17 00:24:16,056 : INFO : L2 norm: 28.02006726219276\n",
- "2019-06-17 00:24:16,108 : INFO : topic #0 (0.345): 0.014*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.007*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"polici\" + 0.003*\"attack\" + 0.003*\"henri\"\n",
- "2019-06-17 00:24:16,109 : INFO : topic #1 (0.253): 0.008*\"space\" + 0.005*\"nasa\" + 0.004*\"access\" + 0.003*\"orbit\" + 0.003*\"pat\" + 0.003*\"digex\" + 0.003*\"launch\" + 0.003*\"shuttl\" + 0.003*\"graphic\" + 0.003*\"com\"\n",
- "2019-06-17 00:24:16,110 : INFO : topic #2 (0.299): 0.020*\"armenian\" + 0.010*\"turkish\" + 0.007*\"armenia\" + 0.006*\"turk\" + 0.006*\"argic\" + 0.006*\"serdar\" + 0.005*\"greek\" + 0.005*\"turkei\" + 0.004*\"genocid\" + 0.004*\"peopl\"\n",
- "2019-06-17 00:24:16,112 : INFO : topic #3 (0.353): 0.013*\"moral\" + 0.011*\"keith\" + 0.006*\"object\" + 0.005*\"caltech\" + 0.005*\"schneider\" + 0.004*\"anim\" + 0.004*\"allan\" + 0.004*\"cco\" + 0.004*\"jake\" + 0.004*\"boni\"\n",
- "2019-06-17 00:24:16,113 : INFO : topic #4 (0.380): 0.011*\"islam\" + 0.011*\"god\" + 0.006*\"livesei\" + 0.006*\"sgi\" + 0.006*\"jaeger\" + 0.005*\"muslim\" + 0.005*\"jon\" + 0.005*\"religion\" + 0.004*\"imag\" + 0.004*\"solntz\"\n",
- "2019-06-17 00:24:16,124 : INFO : W error diff: -0.05304441334403265\n",
- "2019-06-17 00:24:16,155 : INFO : PROGRESS: pass 2, at document #1000/2819\n",
- "2019-06-17 00:24:16,170 : INFO : W error diff: -0.6532464912217009\n",
- "2019-06-17 00:24:16,204 : INFO : PROGRESS: pass 2, at document #2000/2819\n",
- "2019-06-17 00:24:16,212 : INFO : W error diff: -0.5542774416923812\n",
- "2019-06-17 00:24:16,228 : INFO : PROGRESS: pass 2, at document #2819/2819\n",
- "2019-06-17 00:24:16,546 : INFO : L2 norm: 27.999892226543682\n",
- "2019-06-17 00:24:16,604 : INFO : topic #0 (0.343): 0.014*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"polici\" + 0.003*\"lebanon\"\n",
- "2019-06-17 00:24:16,605 : INFO : topic #1 (0.229): 0.007*\"space\" + 0.005*\"nasa\" + 0.004*\"access\" + 0.003*\"orbit\" + 0.003*\"pat\" + 0.003*\"launch\" + 0.003*\"digex\" + 0.003*\"gov\" + 0.003*\"graphic\" + 0.003*\"com\"\n",
- "2019-06-17 00:24:16,607 : INFO : topic #2 (0.283): 0.022*\"armenian\" + 0.011*\"turkish\" + 0.007*\"armenia\" + 0.007*\"turk\" + 0.007*\"argic\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.004*\"soviet\"\n",
- "2019-06-17 00:24:16,609 : INFO : topic #3 (0.347): 0.015*\"moral\" + 0.013*\"keith\" + 0.007*\"object\" + 0.006*\"caltech\" + 0.005*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.004*\"jake\" + 0.004*\"natur\"\n",
- "2019-06-17 00:24:16,610 : INFO : topic #4 (0.365): 0.011*\"god\" + 0.011*\"islam\" + 0.006*\"livesei\" + 0.006*\"sgi\" + 0.006*\"jaeger\" + 0.005*\"muslim\" + 0.005*\"religion\" + 0.005*\"jon\" + 0.005*\"atheist\" + 0.004*\"atheism\"\n",
- "2019-06-17 00:24:16,618 : INFO : W error diff: 0.06399021760879364\n",
- "2019-06-17 00:24:16,659 : INFO : PROGRESS: pass 3, at document #1000/2819\n",
- "2019-06-17 00:24:16,676 : INFO : W error diff: -0.3678424933365889\n",
- "2019-06-17 00:24:16,697 : INFO : PROGRESS: pass 3, at document #2000/2819\n",
- "2019-06-17 00:24:16,705 : INFO : W error diff: -0.34924666183303543\n",
- "2019-06-17 00:24:16,722 : INFO : PROGRESS: pass 3, at document #2819/2819\n",
- "2019-06-17 00:24:17,141 : INFO : L2 norm: 27.991268049236886\n",
- "2019-06-17 00:24:17,190 : INFO : topic #0 (0.350): 0.015*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"lebanon\" + 0.003*\"polici\"\n",
- "2019-06-17 00:24:17,192 : INFO : topic #1 (0.220): 0.007*\"space\" + 0.005*\"nasa\" + 0.003*\"access\" + 0.003*\"orbit\" + 0.003*\"launch\" + 0.003*\"pat\" + 0.003*\"gov\" + 0.003*\"com\" + 0.003*\"digex\" + 0.002*\"alaska\"\n",
- "2019-06-17 00:24:17,194 : INFO : topic #2 (0.282): 0.023*\"armenian\" + 0.011*\"turkish\" + 0.007*\"armenia\" + 0.007*\"turk\" + 0.007*\"argic\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.005*\"soviet\"\n",
- "2019-06-17 00:24:17,196 : INFO : topic #3 (0.351): 0.016*\"moral\" + 0.015*\"keith\" + 0.007*\"object\" + 0.007*\"caltech\" + 0.006*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.004*\"natur\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:17,197 : INFO : topic #4 (0.364): 0.012*\"god\" + 0.011*\"islam\" + 0.006*\"sgi\" + 0.006*\"jaeger\" + 0.006*\"livesei\" + 0.005*\"muslim\" + 0.005*\"religion\" + 0.005*\"atheist\" + 0.005*\"atheism\" + 0.004*\"jon\"\n",
- "2019-06-17 00:24:17,204 : INFO : W error diff: 0.08877110840856872\n",
- "2019-06-17 00:24:17,242 : INFO : PROGRESS: pass 4, at document #1000/2819\n",
- "2019-06-17 00:24:17,257 : INFO : W error diff: -0.2446709705343757\n",
- "2019-06-17 00:24:17,282 : INFO : PROGRESS: pass 4, at document #2000/2819\n",
- "2019-06-17 00:24:17,290 : INFO : W error diff: -0.24931839405260803\n",
- "2019-06-17 00:24:17,307 : INFO : PROGRESS: pass 4, at document #2819/2819\n",
- "2019-06-17 00:24:17,642 : INFO : L2 norm: 27.98648818098989\n",
- "2019-06-17 00:24:17,700 : INFO : topic #0 (0.354): 0.015*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.004*\"attack\" + 0.003*\"lebanon\" + 0.003*\"polici\"\n",
- "2019-06-17 00:24:17,702 : INFO : topic #1 (0.209): 0.007*\"space\" + 0.005*\"nasa\" + 0.003*\"access\" + 0.003*\"orbit\" + 0.003*\"launch\" + 0.003*\"gov\" + 0.003*\"pat\" + 0.003*\"com\" + 0.002*\"alaska\" + 0.002*\"moon\"\n",
- "2019-06-17 00:24:17,704 : INFO : topic #2 (0.283): 0.023*\"armenian\" + 0.011*\"turkish\" + 0.008*\"armenia\" + 0.007*\"argic\" + 0.007*\"turk\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.005*\"soviet\"\n",
- "2019-06-17 00:24:17,705 : INFO : topic #3 (0.356): 0.017*\"moral\" + 0.016*\"keith\" + 0.007*\"object\" + 0.007*\"caltech\" + 0.006*\"schneider\" + 0.006*\"allan\" + 0.006*\"cco\" + 0.004*\"anim\" + 0.004*\"natur\" + 0.004*\"goal\"\n",
- "2019-06-17 00:24:17,707 : INFO : topic #4 (0.366): 0.012*\"god\" + 0.011*\"islam\" + 0.006*\"jaeger\" + 0.005*\"sgi\" + 0.005*\"livesei\" + 0.005*\"muslim\" + 0.005*\"atheist\" + 0.005*\"religion\" + 0.005*\"atheism\" + 0.004*\"rushdi\"\n",
- "2019-06-17 00:24:17,714 : INFO : W error diff: 0.0932956490045207\n"
+ "2019-05-06 15:57:18,565 : INFO : running NMF training, 5 topics, 5 passes over the supplied corpus of 2819 documents, evaluating l2 norm every 2819 documents\n",
+ "2019-05-06 15:57:18,581 : INFO : PROGRESS: pass 0, at document #1000/2819\n",
+ "2019-05-06 15:57:18,604 : INFO : W error: -11.552583824232896\n",
+ "2019-05-06 15:57:18,619 : INFO : PROGRESS: pass 0, at document #2000/2819\n",
+ "2019-05-06 15:57:18,627 : INFO : W error: -13.74803744073488\n",
+ "2019-05-06 15:57:18,639 : INFO : PROGRESS: pass 0, at document #2819/2819\n",
+ "2019-05-06 15:57:18,735 : INFO : L2 norm: 28.141396382337142\n",
+ "2019-05-06 15:57:18,770 : INFO : topic #0 (0.395): 0.011*\"isra\" + 0.010*\"israel\" + 0.006*\"arab\" + 0.006*\"jew\" + 0.004*\"palestinian\" + 0.004*\"henri\" + 0.003*\"toronto\" + 0.003*\"question\" + 0.003*\"kill\" + 0.003*\"hernlem\"\n",
+ "2019-05-06 15:57:18,771 : INFO : topic #1 (0.352): 0.008*\"space\" + 0.005*\"access\" + 0.005*\"nasa\" + 0.004*\"pat\" + 0.003*\"digex\" + 0.003*\"orbit\" + 0.003*\"shuttl\" + 0.003*\"data\" + 0.003*\"graphic\" + 0.003*\"com\"\n",
+ "2019-05-06 15:57:18,772 : INFO : topic #2 (0.378): 0.012*\"armenian\" + 0.006*\"turkish\" + 0.004*\"greek\" + 0.004*\"peopl\" + 0.004*\"armenia\" + 0.004*\"turk\" + 0.004*\"argic\" + 0.004*\"bike\" + 0.003*\"serdar\" + 0.003*\"turkei\"\n",
+ "2019-05-06 15:57:18,772 : INFO : topic #3 (0.412): 0.010*\"moral\" + 0.006*\"keith\" + 0.004*\"anim\" + 0.003*\"jake\" + 0.003*\"boni\" + 0.003*\"instinct\" + 0.003*\"act\" + 0.003*\"think\" + 0.003*\"object\" + 0.003*\"caltech\"\n",
+ "2019-05-06 15:57:18,773 : INFO : topic #4 (0.428): 0.009*\"islam\" + 0.008*\"god\" + 0.006*\"livesei\" + 0.006*\"muslim\" + 0.005*\"imag\" + 0.005*\"sgi\" + 0.005*\"jaeger\" + 0.004*\"jon\" + 0.004*\"solntz\" + 0.004*\"wpd\"\n",
+ "2019-05-06 15:57:18,776 : INFO : W error: -14.346700852834074\n",
+ "2019-05-06 15:57:18,791 : INFO : PROGRESS: pass 1, at document #1000/2819\n",
+ "2019-05-06 15:57:18,797 : INFO : W error: -15.92349117004869\n",
+ "2019-05-06 15:57:18,811 : INFO : PROGRESS: pass 1, at document #2000/2819\n",
+ "2019-05-06 15:57:18,816 : INFO : W error: -17.04437515427273\n",
+ "2019-05-06 15:57:18,829 : INFO : PROGRESS: pass 1, at document #2819/2819\n",
+ "2019-05-06 15:57:18,922 : INFO : L2 norm: 28.021861174968205\n",
+ "2019-05-06 15:57:18,956 : INFO : topic #0 (0.341): 0.014*\"israel\" + 0.013*\"isra\" + 0.008*\"arab\" + 0.007*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.003*\"peac\" + 0.003*\"henri\" + 0.003*\"attack\" + 0.003*\"polici\"\n",
+ "2019-05-06 15:57:18,956 : INFO : topic #1 (0.252): 0.008*\"space\" + 0.005*\"nasa\" + 0.004*\"access\" + 0.003*\"pat\" + 0.003*\"orbit\" + 0.003*\"digex\" + 0.003*\"launch\" + 0.003*\"shuttl\" + 0.003*\"graphic\" + 0.003*\"com\"\n",
+ "2019-05-06 15:57:18,957 : INFO : topic #2 (0.295): 0.020*\"armenian\" + 0.010*\"turkish\" + 0.006*\"armenia\" + 0.006*\"turk\" + 0.006*\"argic\" + 0.006*\"serdar\" + 0.005*\"greek\" + 0.005*\"turkei\" + 0.004*\"peopl\" + 0.004*\"genocid\"\n",
+ "2019-05-06 15:57:18,958 : INFO : topic #3 (0.345): 0.013*\"moral\" + 0.011*\"keith\" + 0.006*\"object\" + 0.005*\"caltech\" + 0.004*\"schneider\" + 0.004*\"anim\" + 0.004*\"jake\" + 0.004*\"allan\" + 0.004*\"boni\" + 0.004*\"cco\"\n",
+ "2019-05-06 15:57:18,959 : INFO : topic #4 (0.375): 0.011*\"islam\" + 0.011*\"god\" + 0.006*\"livesei\" + 0.006*\"sgi\" + 0.006*\"jaeger\" + 0.005*\"muslim\" + 0.005*\"jon\" + 0.005*\"imag\" + 0.005*\"religion\" + 0.005*\"solntz\"\n",
+ "2019-05-06 15:57:18,961 : INFO : W error: -17.08829704968913\n",
+ "2019-05-06 15:57:18,975 : INFO : PROGRESS: pass 2, at document #1000/2819\n",
+ "2019-05-06 15:57:18,981 : INFO : W error: -17.73961065930116\n",
+ "2019-05-06 15:57:18,996 : INFO : PROGRESS: pass 2, at document #2000/2819\n",
+ "2019-05-06 15:57:19,001 : INFO : W error: -18.289085863153712\n",
+ "2019-05-06 15:57:19,013 : INFO : PROGRESS: pass 2, at document #2819/2819\n",
+ "2019-05-06 15:57:19,107 : INFO : L2 norm: 28.001900396967052\n",
+ "2019-05-06 15:57:19,141 : INFO : topic #0 (0.341): 0.014*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"polici\" + 0.003*\"lebanon\"\n",
+ "2019-05-06 15:57:19,142 : INFO : topic #1 (0.230): 0.007*\"space\" + 0.005*\"nasa\" + 0.004*\"access\" + 0.003*\"orbit\" + 0.003*\"pat\" + 0.003*\"launch\" + 0.003*\"digex\" + 0.003*\"gov\" + 0.003*\"com\" + 0.003*\"graphic\"\n",
+ "2019-05-06 15:57:19,142 : INFO : topic #2 (0.284): 0.021*\"armenian\" + 0.011*\"turkish\" + 0.007*\"armenia\" + 0.007*\"turk\" + 0.007*\"argic\" + 0.006*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.004*\"soviet\"\n",
+ "2019-05-06 15:57:19,143 : INFO : topic #3 (0.346): 0.015*\"moral\" + 0.013*\"keith\" + 0.006*\"object\" + 0.006*\"caltech\" + 0.005*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.004*\"jake\" + 0.003*\"boni\"\n",
+ "2019-05-06 15:57:19,144 : INFO : topic #4 (0.365): 0.012*\"islam\" + 0.011*\"god\" + 0.006*\"livesei\" + 0.006*\"sgi\" + 0.006*\"jaeger\" + 0.005*\"muslim\" + 0.005*\"jon\" + 0.005*\"religion\" + 0.004*\"solntz\" + 0.004*\"wpd\"\n",
+ "2019-05-06 15:57:19,146 : INFO : W error: -18.220202313431095\n",
+ "2019-05-06 15:57:19,160 : INFO : PROGRESS: pass 3, at document #1000/2819\n",
+ "2019-05-06 15:57:19,165 : INFO : W error: -18.590446221955172\n",
+ "2019-05-06 15:57:19,180 : INFO : PROGRESS: pass 3, at document #2000/2819\n",
+ "2019-05-06 15:57:19,185 : INFO : W error: -18.936998738726114\n",
+ "2019-05-06 15:57:19,197 : INFO : PROGRESS: pass 3, at document #2819/2819\n",
+ "2019-05-06 15:57:19,291 : INFO : L2 norm: 27.993018072469805\n",
+ "2019-05-06 15:57:19,324 : INFO : topic #0 (0.348): 0.015*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"lebanon\" + 0.003*\"polici\"\n",
+ "2019-05-06 15:57:19,325 : INFO : topic #1 (0.221): 0.007*\"space\" + 0.005*\"nasa\" + 0.003*\"access\" + 0.003*\"orbit\" + 0.003*\"pat\" + 0.003*\"launch\" + 0.003*\"gov\" + 0.003*\"digex\" + 0.003*\"com\" + 0.002*\"graphic\"\n",
+ "2019-05-06 15:57:19,325 : INFO : topic #2 (0.281): 0.022*\"armenian\" + 0.011*\"turkish\" + 0.007*\"armenia\" + 0.007*\"turk\" + 0.007*\"argic\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.005*\"soviet\"\n",
+ "2019-05-06 15:57:19,326 : INFO : topic #3 (0.349): 0.016*\"moral\" + 0.014*\"keith\" + 0.007*\"object\" + 0.006*\"caltech\" + 0.006*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.004*\"natur\" + 0.003*\"think\"\n",
+ "2019-05-06 15:57:19,327 : INFO : topic #4 (0.365): 0.012*\"islam\" + 0.012*\"god\" + 0.006*\"sgi\" + 0.006*\"livesei\" + 0.006*\"jaeger\" + 0.005*\"muslim\" + 0.005*\"religion\" + 0.005*\"atheist\" + 0.005*\"jon\" + 0.005*\"atheism\"\n",
+ "2019-05-06 15:57:19,328 : INFO : W error: -18.84541532652092\n",
+ "2019-05-06 15:57:19,342 : INFO : PROGRESS: pass 4, at document #1000/2819\n",
+ "2019-05-06 15:57:19,347 : INFO : W error: -19.091058241700402\n",
+ "2019-05-06 15:57:19,362 : INFO : PROGRESS: pass 4, at document #2000/2819\n",
+ "2019-05-06 15:57:19,367 : INFO : W error: -19.338453391122066\n",
+ "2019-05-06 15:57:19,378 : INFO : PROGRESS: pass 4, at document #2819/2819\n",
+ "2019-05-06 15:57:19,473 : INFO : L2 norm: 27.988158841345246\n",
+ "2019-05-06 15:57:19,506 : INFO : topic #0 (0.352): 0.015*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"lebanon\" + 0.003*\"polici\"\n",
+ "2019-05-06 15:57:19,507 : INFO : topic #1 (0.210): 0.007*\"space\" + 0.005*\"nasa\" + 0.003*\"access\" + 0.003*\"orbit\" + 0.003*\"launch\" + 0.003*\"pat\" + 0.003*\"gov\" + 0.003*\"com\" + 0.002*\"alaska\" + 0.002*\"graphic\"\n",
+ "2019-05-06 15:57:19,508 : INFO : topic #2 (0.282): 0.023*\"armenian\" + 0.011*\"turkish\" + 0.008*\"armenia\" + 0.007*\"argic\" + 0.007*\"turk\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.005*\"soviet\"\n",
+ "2019-05-06 15:57:19,509 : INFO : topic #3 (0.353): 0.016*\"moral\" + 0.015*\"keith\" + 0.007*\"object\" + 0.007*\"caltech\" + 0.006*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.004*\"natur\" + 0.004*\"goal\"\n",
+ "2019-05-06 15:57:19,509 : INFO : topic #4 (0.367): 0.012*\"god\" + 0.012*\"islam\" + 0.006*\"jaeger\" + 0.006*\"sgi\" + 0.006*\"livesei\" + 0.005*\"muslim\" + 0.005*\"atheist\" + 0.005*\"religion\" + 0.005*\"atheism\" + 0.004*\"jon\"\n",
+ "2019-05-06 15:57:19,511 : INFO : W error: -19.245120389312117\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
- "CPU times: user 7.85 s, sys: 6.88 s, total: 14.7 s\n",
- "Wall time: 2.6 s\n"
+ "CPU times: user 1.52 s, sys: 1.84 s, total: 3.36 s\n",
+ "Wall time: 947 ms\n"
]
}
],
@@ -467,6 +458,68 @@
")"
]
},
+ {
+ "cell_type": "code",
+ "execution_count": 8,
+ "metadata": {},
+ "outputs": [],
+ "source": [
+ "W = nmf.get_topics().T\n",
+ "\n",
+ "dense_test_corpus = matutils.corpus2dense(\n",
+ " test_corpus_tfidf,\n",
+ " num_terms=W.shape[0],\n",
+ ")\n",
+ "\n",
+ "if isinstance(nmf, SklearnNmf):\n",
+ " H = nmf.transform(dense_test_corpus.T).T\n",
+ "else:\n",
+ " H = np.zeros((nmf.num_topics, len(test_corpus_tfidf)))\n",
+ " for bow_id, bow in enumerate(test_corpus_tfidf):\n",
+ " for topic_id, word_count in nmf[bow]:\n",
+ " H[topic_id, bow_id] = word_count"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 9,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "1.105176733465657"
+ ]
+ },
+ "execution_count": 9,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.linalg.norm(W.dot(H))"
+ ]
+ },
+ {
+ "cell_type": "code",
+ "execution_count": 10,
+ "metadata": {},
+ "outputs": [
+ {
+ "data": {
+ "text/plain": [
+ "43.312817"
+ ]
+ },
+ "execution_count": 10,
+ "metadata": {},
+ "output_type": "execute_result"
+ }
+ ],
+ "source": [
+ "np.linalg.norm(dense_test_corpus)"
+ ]
+ },
{
"cell_type": "markdown",
"metadata": {},
@@ -476,7 +529,7 @@
},
{
"cell_type": "code",
- "execution_count": 8,
+ "execution_count": 11,
"metadata": {},
"outputs": [
{
@@ -485,16 +538,16 @@
"[(0,\n",
" '0.015*\"israel\" + 0.014*\"isra\" + 0.009*\"arab\" + 0.008*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.004*\"attack\" + 0.004*\"lebanon\" + 0.003*\"polici\"'),\n",
" (1,\n",
- " '0.007*\"space\" + 0.005*\"nasa\" + 0.003*\"access\" + 0.003*\"orbit\" + 0.003*\"launch\" + 0.003*\"gov\" + 0.003*\"pat\" + 0.003*\"com\" + 0.002*\"alaska\" + 0.002*\"moon\"'),\n",
+ " '0.007*\"space\" + 0.005*\"nasa\" + 0.003*\"access\" + 0.003*\"orbit\" + 0.003*\"launch\" + 0.003*\"pat\" + 0.003*\"gov\" + 0.003*\"com\" + 0.002*\"alaska\" + 0.002*\"moon\"'),\n",
" (2,\n",
- " '0.023*\"armenian\" + 0.012*\"turkish\" + 0.008*\"armenia\" + 0.007*\"argic\" + 0.007*\"turk\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.005*\"soviet\"'),\n",
+ " '0.023*\"armenian\" + 0.011*\"turkish\" + 0.008*\"armenia\" + 0.007*\"argic\" + 0.007*\"turk\" + 0.007*\"serdar\" + 0.006*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.005*\"soviet\"'),\n",
" (3,\n",
- " '0.017*\"moral\" + 0.016*\"keith\" + 0.008*\"object\" + 0.007*\"caltech\" + 0.006*\"schneider\" + 0.006*\"allan\" + 0.006*\"cco\" + 0.004*\"anim\" + 0.004*\"natur\" + 0.004*\"goal\"'),\n",
+ " '0.016*\"moral\" + 0.015*\"keith\" + 0.007*\"object\" + 0.007*\"caltech\" + 0.006*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.004*\"natur\" + 0.004*\"goal\"'),\n",
" (4,\n",
- " '0.012*\"god\" + 0.011*\"islam\" + 0.006*\"jaeger\" + 0.005*\"sgi\" + 0.005*\"livesei\" + 0.005*\"muslim\" + 0.005*\"atheist\" + 0.005*\"religion\" + 0.005*\"atheism\" + 0.004*\"rushdi\"')]"
+ " '0.012*\"god\" + 0.012*\"islam\" + 0.006*\"jaeger\" + 0.006*\"sgi\" + 0.005*\"livesei\" + 0.005*\"muslim\" + 0.005*\"atheist\" + 0.005*\"religion\" + 0.005*\"atheism\" + 0.004*\"rushdi\"')]"
]
},
- "execution_count": 8,
+ "execution_count": 11,
"metadata": {},
"output_type": "execute_result"
}
@@ -514,23 +567,23 @@
},
{
"cell_type": "code",
- "execution_count": 9,
+ "execution_count": 12,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
- "2019-06-17 00:24:17,821 : INFO : CorpusAccumulator accumulated stats from 1000 documents\n"
+ "2019-05-06 15:57:20,590 : INFO : CorpusAccumulator accumulated stats from 1000 documents\n"
]
},
{
"data": {
"text/plain": [
- "-4.045883079644641"
+ "-4.1310114675795875"
]
},
- "execution_count": 9,
+ "execution_count": 12,
"metadata": {},
"output_type": "execute_result"
}
@@ -556,7 +609,7 @@
},
{
"cell_type": "code",
- "execution_count": 10,
+ "execution_count": 13,
"metadata": {},
"outputs": [
{
@@ -590,7 +643,7 @@
"Convulsed, foaming immortal blood: farewell\" - J. Berryman, \"A Professor's Song\"\n",
"\n",
"====================================================================================================\n",
- "Topics: [(0, 0.10094349983895379), (1, 0.40527834628482196), (2, 0.14330724750919113), (3, 0.02887286985628184), (4, 0.32159803651075125)]\n"
+ "Topics: [(0, 0.10199317206513686), (1, 0.39976628221371285), (2, 0.1428263926167706), (3, 0.0333080734922002), (4, 0.3221060796121796)]\n"
]
}
],
@@ -611,7 +664,7 @@
},
{
"cell_type": "code",
- "execution_count": 11,
+ "execution_count": 14,
"metadata": {
"lines_to_next_cell": 2
},
@@ -621,7 +674,7 @@
"output_type": "stream",
"text": [
"Word: actual\n",
- "Topics: [(0, 0.1517466731147538), (1, 0.2824521057319929), (2, 0.042590027339691805), (3, 0.2520757387076886), (4, 0.2711354551058729)]\n"
+ "Topics: [(0, 0.15401782844659068), (1, 0.2829834256007429), (2, 0.04354905106817273), (3, 0.25783766798021135), (4, 0.26161202690428226)]\n"
]
}
],
@@ -647,7 +700,7 @@
},
{
"cell_type": "code",
- "execution_count": 12,
+ "execution_count": 15,
"metadata": {
"lines_to_next_cell": 2
},
@@ -666,14 +719,14 @@
},
{
"cell_type": "code",
- "execution_count": 13,
+ "execution_count": 16,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Density: 0.6864567151532269\n"
+ "Density: 0.6872475639034035\n"
]
}
],
@@ -690,14 +743,14 @@
},
{
"cell_type": "code",
- "execution_count": 14,
+ "execution_count": 17,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
- "Density: 0.662026862026862\n"
+ "Density: 0.6615384615384615\n"
]
}
],
@@ -741,7 +794,7 @@
},
{
"cell_type": "code",
- "execution_count": 15,
+ "execution_count": 18,
"metadata": {
"lines_to_next_cell": 2
},
@@ -760,7 +813,7 @@
},
{
"cell_type": "code",
- "execution_count": 16,
+ "execution_count": 19,
"metadata": {},
"outputs": [],
"source": [
@@ -790,7 +843,7 @@
" with measure_ram(memprof_filename, tick=tick):\n",
" result = func()\n",
"\n",
- " elapsed_time = pd.to_timedelta(time.time() - start, unit='s').round('s')\n",
+ " elapsed_time = pd.to_timedelta(time.time() - start, unit='s').round('ms')\n",
"\n",
" memprof_df = pd.read_csv(memprof_filename, squeeze=True)\n",
"\n",
@@ -813,7 +866,7 @@
" else:\n",
" X_train = np.zeros((len(train_corpus), model.num_topics))\n",
" for bow_id, bow in enumerate(train_corpus):\n",
- " for topic_id, word_count in model.get_document_topics(bow):\n",
+ " for topic_id, word_count in model[bow]:\n",
" X_train[bow_id, topic_id] = word_count\n",
"\n",
" log_reg = LogisticRegressionCV(multi_class='multinomial', cv=5)\n",
@@ -876,7 +929,7 @@
" else:\n",
" H = np.zeros((model.num_topics, len(test_corpus)))\n",
" for bow_id, bow in enumerate(test_corpus):\n",
- " for topic_id, word_count in model.get_document_topics(bow):\n",
+ " for topic_id, word_count in model[bow]:\n",
" H[topic_id, bow_id] = word_count\n",
"\n",
" l2_norm = None\n",
@@ -923,243 +976,11 @@
},
{
"cell_type": "code",
- "execution_count": 17,
+ "execution_count": null,
"metadata": {
"scrolled": true
},
- "outputs": [
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2019-06-17 00:24:17,972 : INFO : using symmetric alpha at 0.2\n",
- "2019-06-17 00:24:17,975 : INFO : using symmetric eta at 0.2\n",
- "2019-06-17 00:24:17,977 : INFO : using serial LDA version on this node\n",
- "2019-06-17 00:24:17,986 : INFO : running online (multi-pass) LDA training, 5 topics, 5 passes over the supplied corpus of 2819 documents, updating model once every 1000 documents, evaluating perplexity every 2819 documents, iterating 50x with a convergence threshold of 0.001000\n",
- "2019-06-17 00:24:17,989 : INFO : PROGRESS: pass 0, at document #1000/2819\n",
- "2019-06-17 00:24:18,934 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:18,939 : INFO : topic #0 (0.200): 0.006*\"com\" + 0.006*\"like\" + 0.006*\"space\" + 0.004*\"univers\" + 0.004*\"know\" + 0.004*\"nntp\" + 0.004*\"imag\" + 0.004*\"nasa\" + 0.004*\"time\" + 0.004*\"program\"\n",
- "2019-06-17 00:24:18,940 : INFO : topic #1 (0.200): 0.007*\"imag\" + 0.005*\"com\" + 0.004*\"peopl\" + 0.004*\"know\" + 0.004*\"univers\" + 0.004*\"nntp\" + 0.004*\"host\" + 0.004*\"jpeg\" + 0.004*\"good\" + 0.004*\"work\"\n",
- "2019-06-17 00:24:18,940 : INFO : topic #2 (0.200): 0.007*\"peopl\" + 0.005*\"com\" + 0.005*\"like\" + 0.004*\"new\" + 0.004*\"know\" + 0.004*\"think\" + 0.004*\"nntp\" + 0.003*\"isra\" + 0.003*\"host\" + 0.003*\"question\"\n",
- "2019-06-17 00:24:18,942 : INFO : topic #3 (0.200): 0.008*\"com\" + 0.006*\"space\" + 0.005*\"peopl\" + 0.005*\"like\" + 0.004*\"israel\" + 0.004*\"isra\" + 0.004*\"nasa\" + 0.004*\"think\" + 0.003*\"time\" + 0.003*\"right\"\n",
- "2019-06-17 00:24:18,943 : INFO : topic #4 (0.200): 0.006*\"com\" + 0.006*\"time\" + 0.005*\"host\" + 0.005*\"univers\" + 0.005*\"peopl\" + 0.005*\"nntp\" + 0.004*\"know\" + 0.004*\"god\" + 0.004*\"like\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:18,944 : INFO : topic diff=1.680969, rho=1.000000\n",
- "2019-06-17 00:24:18,947 : INFO : PROGRESS: pass 0, at document #2000/2819\n",
- "2019-06-17 00:24:19,806 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:19,811 : INFO : topic #0 (0.200): 0.008*\"space\" + 0.007*\"com\" + 0.005*\"imag\" + 0.005*\"like\" + 0.005*\"nasa\" + 0.005*\"program\" + 0.004*\"univers\" + 0.004*\"graphic\" + 0.004*\"data\" + 0.004*\"know\"\n",
- "2019-06-17 00:24:19,812 : INFO : topic #1 (0.200): 0.009*\"imag\" + 0.006*\"com\" + 0.005*\"know\" + 0.004*\"univers\" + 0.004*\"peopl\" + 0.004*\"like\" + 0.004*\"work\" + 0.004*\"want\" + 0.004*\"armenian\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:19,813 : INFO : topic #2 (0.200): 0.012*\"peopl\" + 0.007*\"know\" + 0.006*\"armenian\" + 0.006*\"said\" + 0.006*\"like\" + 0.005*\"think\" + 0.004*\"right\" + 0.004*\"time\" + 0.004*\"wai\" + 0.004*\"thing\"\n",
- "2019-06-17 00:24:19,814 : INFO : topic #3 (0.200): 0.008*\"com\" + 0.007*\"space\" + 0.006*\"israel\" + 0.005*\"peopl\" + 0.005*\"like\" + 0.005*\"isra\" + 0.004*\"nasa\" + 0.004*\"time\" + 0.003*\"think\" + 0.003*\"right\"\n",
- "2019-06-17 00:24:19,816 : INFO : topic #4 (0.200): 0.008*\"god\" + 0.006*\"univers\" + 0.006*\"armenian\" + 0.005*\"com\" + 0.005*\"time\" + 0.005*\"peopl\" + 0.005*\"host\" + 0.004*\"know\" + 0.004*\"nntp\" + 0.004*\"like\"\n",
- "2019-06-17 00:24:19,817 : INFO : topic diff=0.838420, rho=0.707107\n",
- "2019-06-17 00:24:20,803 : INFO : -8.089 per-word bound, 272.3 perplexity estimate based on a held-out corpus of 819 documents with 113268 words\n",
- "2019-06-17 00:24:20,804 : INFO : PROGRESS: pass 0, at document #2819/2819\n",
- "2019-06-17 00:24:21,432 : INFO : merging changes from 819 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:21,436 : INFO : topic #0 (0.200): 0.009*\"space\" + 0.008*\"com\" + 0.006*\"graphic\" + 0.005*\"program\" + 0.005*\"imag\" + 0.004*\"nasa\" + 0.004*\"like\" + 0.004*\"univers\" + 0.004*\"file\" + 0.004*\"new\"\n",
- "2019-06-17 00:24:21,437 : INFO : topic #1 (0.200): 0.007*\"com\" + 0.007*\"imag\" + 0.005*\"univers\" + 0.005*\"know\" + 0.004*\"work\" + 0.004*\"like\" + 0.004*\"host\" + 0.004*\"think\" + 0.004*\"nntp\" + 0.004*\"moral\"\n",
- "2019-06-17 00:24:21,437 : INFO : topic #2 (0.200): 0.011*\"peopl\" + 0.007*\"armenian\" + 0.006*\"said\" + 0.006*\"turkish\" + 0.006*\"know\" + 0.005*\"think\" + 0.005*\"like\" + 0.005*\"right\" + 0.005*\"jew\" + 0.004*\"isra\"\n",
- "2019-06-17 00:24:21,438 : INFO : topic #3 (0.200): 0.008*\"com\" + 0.008*\"israel\" + 0.006*\"space\" + 0.005*\"bike\" + 0.005*\"isra\" + 0.005*\"like\" + 0.004*\"peopl\" + 0.004*\"year\" + 0.004*\"jew\" + 0.004*\"time\"\n",
- "2019-06-17 00:24:21,439 : INFO : topic #4 (0.200): 0.008*\"god\" + 0.007*\"armenian\" + 0.006*\"univers\" + 0.005*\"peopl\" + 0.005*\"com\" + 0.005*\"exist\" + 0.005*\"time\" + 0.004*\"like\" + 0.004*\"think\" + 0.004*\"know\"\n",
- "2019-06-17 00:24:21,440 : INFO : topic diff=0.645193, rho=0.577350\n",
- "2019-06-17 00:24:21,441 : INFO : PROGRESS: pass 1, at document #1000/2819\n",
- "2019-06-17 00:24:22,110 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:22,114 : INFO : topic #0 (0.200): 0.010*\"space\" + 0.007*\"com\" + 0.006*\"graphic\" + 0.006*\"nasa\" + 0.005*\"program\" + 0.005*\"imag\" + 0.005*\"file\" + 0.004*\"data\" + 0.004*\"univers\" + 0.004*\"like\"\n",
- "2019-06-17 00:24:22,115 : INFO : topic #1 (0.200): 0.008*\"imag\" + 0.008*\"com\" + 0.005*\"univers\" + 0.005*\"know\" + 0.005*\"like\" + 0.005*\"host\" + 0.005*\"bit\" + 0.005*\"nntp\" + 0.004*\"work\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:22,116 : INFO : topic #2 (0.200): 0.011*\"peopl\" + 0.007*\"armenian\" + 0.006*\"said\" + 0.006*\"right\" + 0.006*\"turkish\" + 0.005*\"know\" + 0.005*\"isra\" + 0.005*\"think\" + 0.005*\"like\" + 0.005*\"jew\"\n",
- "2019-06-17 00:24:22,117 : INFO : topic #3 (0.200): 0.009*\"com\" + 0.008*\"israel\" + 0.006*\"bike\" + 0.006*\"isra\" + 0.005*\"space\" + 0.005*\"like\" + 0.004*\"peopl\" + 0.004*\"year\" + 0.004*\"think\" + 0.004*\"time\"\n",
- "2019-06-17 00:24:22,118 : INFO : topic #4 (0.200): 0.010*\"god\" + 0.007*\"armenian\" + 0.006*\"peopl\" + 0.005*\"univers\" + 0.005*\"com\" + 0.005*\"time\" + 0.005*\"exist\" + 0.004*\"islam\" + 0.004*\"believ\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:22,119 : INFO : topic diff=0.427516, rho=0.455535\n",
- "2019-06-17 00:24:22,120 : INFO : PROGRESS: pass 1, at document #2000/2819\n",
- "2019-06-17 00:24:22,819 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:22,823 : INFO : topic #0 (0.200): 0.011*\"space\" + 0.007*\"imag\" + 0.007*\"com\" + 0.006*\"nasa\" + 0.006*\"program\" + 0.005*\"graphic\" + 0.005*\"data\" + 0.004*\"file\" + 0.004*\"univers\" + 0.004*\"new\"\n",
- "2019-06-17 00:24:22,824 : INFO : topic #1 (0.200): 0.009*\"com\" + 0.007*\"imag\" + 0.005*\"like\" + 0.005*\"know\" + 0.005*\"univers\" + 0.005*\"host\" + 0.005*\"nntp\" + 0.004*\"think\" + 0.004*\"moral\" + 0.004*\"work\"\n",
- "2019-06-17 00:24:22,824 : INFO : topic #2 (0.200): 0.013*\"peopl\" + 0.009*\"armenian\" + 0.007*\"said\" + 0.007*\"know\" + 0.006*\"right\" + 0.005*\"like\" + 0.005*\"think\" + 0.005*\"kill\" + 0.005*\"turkish\" + 0.004*\"jew\"\n",
- "2019-06-17 00:24:22,825 : INFO : topic #3 (0.200): 0.009*\"com\" + 0.009*\"israel\" + 0.006*\"isra\" + 0.006*\"bike\" + 0.005*\"like\" + 0.005*\"space\" + 0.004*\"state\" + 0.004*\"think\" + 0.004*\"peopl\" + 0.004*\"host\"\n",
- "2019-06-17 00:24:22,827 : INFO : topic #4 (0.200): 0.010*\"god\" + 0.007*\"armenian\" + 0.006*\"peopl\" + 0.006*\"univers\" + 0.005*\"exist\" + 0.005*\"islam\" + 0.005*\"com\" + 0.005*\"believ\" + 0.005*\"time\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:22,827 : INFO : topic diff=0.427394, rho=0.455535\n",
- "2019-06-17 00:24:23,767 : INFO : -7.871 per-word bound, 234.1 perplexity estimate based on a held-out corpus of 819 documents with 113268 words\n",
- "2019-06-17 00:24:23,768 : INFO : PROGRESS: pass 1, at document #2819/2819\n",
- "2019-06-17 00:24:24,303 : INFO : merging changes from 819 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:24,307 : INFO : topic #0 (0.200): 0.011*\"space\" + 0.007*\"com\" + 0.006*\"graphic\" + 0.006*\"imag\" + 0.006*\"program\" + 0.005*\"nasa\" + 0.004*\"file\" + 0.004*\"launch\" + 0.004*\"univers\" + 0.004*\"new\"\n",
- "2019-06-17 00:24:24,308 : INFO : topic #1 (0.200): 0.010*\"com\" + 0.006*\"like\" + 0.006*\"univers\" + 0.005*\"imag\" + 0.005*\"know\" + 0.005*\"host\" + 0.005*\"nntp\" + 0.005*\"think\" + 0.005*\"work\" + 0.004*\"henri\"\n",
- "2019-06-17 00:24:24,312 : INFO : topic #2 (0.200): 0.012*\"peopl\" + 0.009*\"armenian\" + 0.008*\"turkish\" + 0.007*\"said\" + 0.006*\"jew\" + 0.006*\"right\" + 0.006*\"know\" + 0.005*\"kill\" + 0.005*\"like\" + 0.005*\"isra\"\n",
- "2019-06-17 00:24:24,314 : INFO : topic #3 (0.200): 0.010*\"com\" + 0.009*\"israel\" + 0.008*\"bike\" + 0.006*\"isra\" + 0.005*\"like\" + 0.005*\"dod\" + 0.004*\"motorcycl\" + 0.004*\"think\" + 0.004*\"state\" + 0.004*\"year\"\n",
- "2019-06-17 00:24:24,316 : INFO : topic #4 (0.200): 0.010*\"god\" + 0.007*\"armenian\" + 0.006*\"peopl\" + 0.006*\"exist\" + 0.006*\"univers\" + 0.005*\"com\" + 0.005*\"think\" + 0.005*\"believ\" + 0.004*\"islam\" + 0.004*\"like\"\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2019-06-17 00:24:24,317 : INFO : topic diff=0.416347, rho=0.455535\n",
- "2019-06-17 00:24:24,318 : INFO : PROGRESS: pass 2, at document #1000/2819\n",
- "2019-06-17 00:24:24,982 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:24,986 : INFO : topic #0 (0.200): 0.011*\"space\" + 0.007*\"imag\" + 0.006*\"nasa\" + 0.006*\"com\" + 0.006*\"graphic\" + 0.006*\"program\" + 0.005*\"file\" + 0.005*\"data\" + 0.005*\"orbit\" + 0.004*\"new\"\n",
- "2019-06-17 00:24:24,987 : INFO : topic #1 (0.200): 0.010*\"com\" + 0.006*\"imag\" + 0.006*\"like\" + 0.006*\"nntp\" + 0.006*\"host\" + 0.006*\"univers\" + 0.005*\"bit\" + 0.005*\"know\" + 0.005*\"think\" + 0.005*\"work\"\n",
- "2019-06-17 00:24:24,988 : INFO : topic #2 (0.200): 0.011*\"peopl\" + 0.009*\"armenian\" + 0.007*\"turkish\" + 0.007*\"said\" + 0.007*\"right\" + 0.006*\"jew\" + 0.005*\"know\" + 0.005*\"kill\" + 0.005*\"isra\" + 0.005*\"like\"\n",
- "2019-06-17 00:24:24,989 : INFO : topic #3 (0.200): 0.010*\"com\" + 0.010*\"israel\" + 0.008*\"bike\" + 0.006*\"isra\" + 0.005*\"like\" + 0.004*\"dod\" + 0.004*\"host\" + 0.004*\"think\" + 0.004*\"nntp\" + 0.004*\"state\"\n",
- "2019-06-17 00:24:24,990 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.007*\"peopl\" + 0.006*\"armenian\" + 0.006*\"exist\" + 0.005*\"univers\" + 0.005*\"islam\" + 0.005*\"com\" + 0.005*\"think\" + 0.005*\"believ\" + 0.005*\"atheist\"\n",
- "2019-06-17 00:24:24,991 : INFO : topic diff=0.332567, rho=0.414549\n",
- "2019-06-17 00:24:24,991 : INFO : PROGRESS: pass 2, at document #2000/2819\n",
- "2019-06-17 00:24:25,619 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:25,623 : INFO : topic #0 (0.200): 0.012*\"space\" + 0.008*\"imag\" + 0.007*\"nasa\" + 0.006*\"com\" + 0.006*\"program\" + 0.006*\"graphic\" + 0.005*\"data\" + 0.005*\"file\" + 0.004*\"new\" + 0.004*\"univers\"\n",
- "2019-06-17 00:24:25,624 : INFO : topic #1 (0.200): 0.011*\"com\" + 0.007*\"like\" + 0.006*\"know\" + 0.006*\"nntp\" + 0.006*\"host\" + 0.006*\"univers\" + 0.005*\"imag\" + 0.005*\"bit\" + 0.005*\"think\" + 0.005*\"henri\"\n",
- "2019-06-17 00:24:25,625 : INFO : topic #2 (0.200): 0.013*\"peopl\" + 0.010*\"armenian\" + 0.008*\"said\" + 0.007*\"know\" + 0.006*\"right\" + 0.006*\"turkish\" + 0.006*\"kill\" + 0.006*\"jew\" + 0.005*\"like\" + 0.005*\"think\"\n",
- "2019-06-17 00:24:25,627 : INFO : topic #3 (0.200): 0.010*\"com\" + 0.009*\"israel\" + 0.008*\"bike\" + 0.007*\"isra\" + 0.005*\"like\" + 0.005*\"host\" + 0.004*\"nntp\" + 0.004*\"state\" + 0.004*\"think\" + 0.004*\"dod\"\n",
- "2019-06-17 00:24:25,629 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.007*\"peopl\" + 0.006*\"exist\" + 0.006*\"univers\" + 0.005*\"armenian\" + 0.005*\"islam\" + 0.005*\"believ\" + 0.005*\"think\" + 0.005*\"com\" + 0.005*\"atheist\"\n",
- "2019-06-17 00:24:25,630 : INFO : topic diff=0.326337, rho=0.414549\n",
- "2019-06-17 00:24:26,456 : INFO : -7.813 per-word bound, 224.8 perplexity estimate based on a held-out corpus of 819 documents with 113268 words\n",
- "2019-06-17 00:24:26,457 : INFO : PROGRESS: pass 2, at document #2819/2819\n",
- "2019-06-17 00:24:26,932 : INFO : merging changes from 819 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:26,936 : INFO : topic #0 (0.200): 0.012*\"space\" + 0.007*\"imag\" + 0.006*\"graphic\" + 0.006*\"nasa\" + 0.006*\"program\" + 0.006*\"com\" + 0.005*\"launch\" + 0.005*\"file\" + 0.004*\"new\" + 0.004*\"univers\"\n",
- "2019-06-17 00:24:26,937 : INFO : topic #1 (0.200): 0.012*\"com\" + 0.006*\"like\" + 0.006*\"nntp\" + 0.006*\"host\" + 0.006*\"univers\" + 0.006*\"know\" + 0.005*\"think\" + 0.005*\"work\" + 0.005*\"henri\" + 0.005*\"bit\"\n",
- "2019-06-17 00:24:26,938 : INFO : topic #2 (0.200): 0.012*\"peopl\" + 0.011*\"armenian\" + 0.008*\"turkish\" + 0.007*\"said\" + 0.007*\"jew\" + 0.006*\"right\" + 0.006*\"know\" + 0.005*\"kill\" + 0.005*\"isra\" + 0.005*\"like\"\n",
- "2019-06-17 00:24:26,939 : INFO : topic #3 (0.200): 0.011*\"com\" + 0.010*\"israel\" + 0.009*\"bike\" + 0.006*\"isra\" + 0.006*\"dod\" + 0.005*\"like\" + 0.005*\"motorcycl\" + 0.004*\"host\" + 0.004*\"ride\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:26,939 : INFO : topic #4 (0.200): 0.010*\"god\" + 0.007*\"peopl\" + 0.006*\"exist\" + 0.006*\"univers\" + 0.005*\"think\" + 0.005*\"armenian\" + 0.005*\"com\" + 0.005*\"believ\" + 0.005*\"islam\" + 0.005*\"christian\"\n",
- "2019-06-17 00:24:26,940 : INFO : topic diff=0.317634, rho=0.414549\n",
- "2019-06-17 00:24:26,941 : INFO : PROGRESS: pass 3, at document #1000/2819\n",
- "2019-06-17 00:24:27,495 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:27,499 : INFO : topic #0 (0.200): 0.012*\"space\" + 0.007*\"imag\" + 0.007*\"nasa\" + 0.006*\"graphic\" + 0.006*\"program\" + 0.005*\"com\" + 0.005*\"file\" + 0.005*\"orbit\" + 0.005*\"data\" + 0.004*\"new\"\n",
- "2019-06-17 00:24:27,500 : INFO : topic #1 (0.200): 0.012*\"com\" + 0.007*\"like\" + 0.006*\"nntp\" + 0.006*\"host\" + 0.006*\"univers\" + 0.005*\"know\" + 0.005*\"bit\" + 0.005*\"think\" + 0.005*\"work\" + 0.005*\"time\"\n",
- "2019-06-17 00:24:27,501 : INFO : topic #2 (0.200): 0.011*\"peopl\" + 0.011*\"armenian\" + 0.008*\"turkish\" + 0.007*\"said\" + 0.007*\"jew\" + 0.007*\"right\" + 0.006*\"kill\" + 0.005*\"know\" + 0.005*\"isra\" + 0.005*\"arab\"\n",
- "2019-06-17 00:24:27,502 : INFO : topic #3 (0.200): 0.011*\"com\" + 0.010*\"israel\" + 0.008*\"bike\" + 0.007*\"isra\" + 0.005*\"like\" + 0.005*\"dod\" + 0.005*\"host\" + 0.005*\"nntp\" + 0.004*\"think\" + 0.004*\"motorcycl\"\n",
- "2019-06-17 00:24:27,503 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.007*\"peopl\" + 0.006*\"exist\" + 0.006*\"think\" + 0.005*\"islam\" + 0.005*\"atheist\" + 0.005*\"univers\" + 0.005*\"believ\" + 0.005*\"com\" + 0.005*\"religion\"\n",
- "2019-06-17 00:24:27,503 : INFO : topic diff=0.252457, rho=0.382948\n",
- "2019-06-17 00:24:27,504 : INFO : PROGRESS: pass 3, at document #2000/2819\n",
- "2019-06-17 00:24:28,064 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:28,067 : INFO : topic #0 (0.200): 0.013*\"space\" + 0.008*\"imag\" + 0.007*\"nasa\" + 0.006*\"program\" + 0.006*\"graphic\" + 0.005*\"com\" + 0.005*\"file\" + 0.005*\"data\" + 0.005*\"new\" + 0.004*\"orbit\"\n",
- "2019-06-17 00:24:28,069 : INFO : topic #1 (0.200): 0.013*\"com\" + 0.007*\"like\" + 0.006*\"nntp\" + 0.006*\"host\" + 0.006*\"know\" + 0.006*\"univers\" + 0.005*\"bit\" + 0.005*\"henri\" + 0.005*\"think\" + 0.004*\"time\"\n",
- "2019-06-17 00:24:28,069 : INFO : topic #2 (0.200): 0.012*\"peopl\" + 0.012*\"armenian\" + 0.008*\"said\" + 0.007*\"turkish\" + 0.006*\"know\" + 0.006*\"right\" + 0.006*\"jew\" + 0.006*\"kill\" + 0.005*\"like\" + 0.005*\"isra\"\n",
- "2019-06-17 00:24:28,071 : INFO : topic #3 (0.200): 0.011*\"com\" + 0.010*\"israel\" + 0.008*\"bike\" + 0.007*\"isra\" + 0.005*\"like\" + 0.005*\"host\" + 0.005*\"nntp\" + 0.005*\"dod\" + 0.005*\"state\" + 0.004*\"think\"\n",
- "2019-06-17 00:24:28,072 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.007*\"peopl\" + 0.006*\"exist\" + 0.006*\"islam\" + 0.006*\"univers\" + 0.006*\"think\" + 0.005*\"believ\" + 0.005*\"com\" + 0.005*\"atheist\" + 0.004*\"like\"\n",
- "2019-06-17 00:24:28,072 : INFO : topic diff=0.249839, rho=0.382948\n",
- "2019-06-17 00:24:28,840 : INFO : -7.780 per-word bound, 219.8 perplexity estimate based on a held-out corpus of 819 documents with 113268 words\n",
- "2019-06-17 00:24:28,841 : INFO : PROGRESS: pass 3, at document #2819/2819\n",
- "2019-06-17 00:24:29,291 : INFO : merging changes from 819 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:29,295 : INFO : topic #0 (0.200): 0.012*\"space\" + 0.007*\"imag\" + 0.006*\"nasa\" + 0.006*\"graphic\" + 0.006*\"program\" + 0.005*\"com\" + 0.005*\"launch\" + 0.005*\"file\" + 0.004*\"new\" + 0.004*\"orbit\"\n",
- "2019-06-17 00:24:29,295 : INFO : topic #1 (0.200): 0.014*\"com\" + 0.007*\"like\" + 0.007*\"nntp\" + 0.006*\"host\" + 0.006*\"know\" + 0.006*\"univers\" + 0.005*\"think\" + 0.005*\"henri\" + 0.005*\"bit\" + 0.005*\"work\"\n",
- "2019-06-17 00:24:29,296 : INFO : topic #2 (0.200): 0.013*\"armenian\" + 0.012*\"peopl\" + 0.009*\"turkish\" + 0.007*\"jew\" + 0.007*\"said\" + 0.006*\"right\" + 0.005*\"know\" + 0.005*\"kill\" + 0.005*\"isra\" + 0.005*\"turkei\"\n",
- "2019-06-17 00:24:29,297 : INFO : topic #3 (0.200): 0.011*\"com\" + 0.010*\"israel\" + 0.009*\"bike\" + 0.006*\"isra\" + 0.006*\"dod\" + 0.005*\"like\" + 0.005*\"ride\" + 0.005*\"motorcycl\" + 0.005*\"host\" + 0.005*\"nntp\"\n",
- "2019-06-17 00:24:29,299 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.008*\"peopl\" + 0.006*\"exist\" + 0.006*\"think\" + 0.006*\"univers\" + 0.005*\"com\" + 0.005*\"believ\" + 0.005*\"islam\" + 0.005*\"christian\" + 0.005*\"atheist\"\n",
- "2019-06-17 00:24:29,299 : INFO : topic diff=0.247979, rho=0.382948\n",
- "2019-06-17 00:24:29,300 : INFO : PROGRESS: pass 4, at document #1000/2819\n",
- "2019-06-17 00:24:29,848 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:29,851 : INFO : topic #0 (0.200): 0.013*\"space\" + 0.008*\"imag\" + 0.007*\"nasa\" + 0.006*\"program\" + 0.006*\"graphic\" + 0.005*\"orbit\" + 0.005*\"file\" + 0.005*\"com\" + 0.005*\"data\" + 0.005*\"new\"\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2019-06-17 00:24:29,852 : INFO : topic #1 (0.200): 0.013*\"com\" + 0.007*\"like\" + 0.007*\"nntp\" + 0.007*\"host\" + 0.006*\"univers\" + 0.006*\"know\" + 0.005*\"bit\" + 0.005*\"think\" + 0.005*\"time\" + 0.005*\"work\"\n",
- "2019-06-17 00:24:29,853 : INFO : topic #2 (0.200): 0.013*\"armenian\" + 0.011*\"peopl\" + 0.008*\"turkish\" + 0.007*\"said\" + 0.007*\"jew\" + 0.006*\"right\" + 0.006*\"kill\" + 0.005*\"know\" + 0.005*\"isra\" + 0.005*\"arab\"\n",
- "2019-06-17 00:24:29,855 : INFO : topic #3 (0.200): 0.012*\"com\" + 0.010*\"israel\" + 0.009*\"bike\" + 0.007*\"isra\" + 0.006*\"dod\" + 0.005*\"like\" + 0.005*\"host\" + 0.005*\"nntp\" + 0.005*\"ride\" + 0.005*\"think\"\n",
- "2019-06-17 00:24:29,856 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.008*\"peopl\" + 0.006*\"think\" + 0.006*\"exist\" + 0.006*\"islam\" + 0.005*\"atheist\" + 0.005*\"believ\" + 0.005*\"com\" + 0.005*\"univers\" + 0.005*\"moral\"\n",
- "2019-06-17 00:24:29,856 : INFO : topic diff=0.202108, rho=0.357622\n",
- "2019-06-17 00:24:29,857 : INFO : PROGRESS: pass 4, at document #2000/2819\n",
- "2019-06-17 00:24:30,410 : INFO : merging changes from 1000 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:30,414 : INFO : topic #0 (0.200): 0.013*\"space\" + 0.009*\"imag\" + 0.007*\"nasa\" + 0.006*\"program\" + 0.006*\"graphic\" + 0.005*\"file\" + 0.005*\"data\" + 0.005*\"com\" + 0.005*\"orbit\" + 0.005*\"new\"\n",
- "2019-06-17 00:24:30,415 : INFO : topic #1 (0.200): 0.014*\"com\" + 0.008*\"like\" + 0.007*\"nntp\" + 0.007*\"host\" + 0.006*\"know\" + 0.006*\"univers\" + 0.005*\"bit\" + 0.005*\"henri\" + 0.005*\"time\" + 0.005*\"think\"\n",
- "2019-06-17 00:24:30,416 : INFO : topic #2 (0.200): 0.013*\"armenian\" + 0.012*\"peopl\" + 0.008*\"said\" + 0.007*\"turkish\" + 0.006*\"jew\" + 0.006*\"know\" + 0.006*\"right\" + 0.006*\"kill\" + 0.005*\"like\" + 0.005*\"isra\"\n",
- "2019-06-17 00:24:30,417 : INFO : topic #3 (0.200): 0.011*\"com\" + 0.010*\"israel\" + 0.008*\"bike\" + 0.007*\"isra\" + 0.006*\"host\" + 0.005*\"like\" + 0.005*\"nntp\" + 0.005*\"dod\" + 0.005*\"state\" + 0.005*\"think\"\n",
- "2019-06-17 00:24:30,418 : INFO : topic #4 (0.200): 0.012*\"god\" + 0.008*\"peopl\" + 0.006*\"think\" + 0.006*\"exist\" + 0.006*\"islam\" + 0.006*\"univers\" + 0.005*\"believ\" + 0.005*\"com\" + 0.005*\"atheist\" + 0.005*\"moral\"\n",
- "2019-06-17 00:24:30,419 : INFO : topic diff=0.201499, rho=0.357622\n",
- "2019-06-17 00:24:31,195 : INFO : -7.758 per-word bound, 216.5 perplexity estimate based on a held-out corpus of 819 documents with 113268 words\n",
- "2019-06-17 00:24:31,195 : INFO : PROGRESS: pass 4, at document #2819/2819\n",
- "2019-06-17 00:24:31,621 : INFO : merging changes from 819 documents into a model of 2819 documents\n",
- "2019-06-17 00:24:31,626 : INFO : topic #0 (0.200): 0.013*\"space\" + 0.008*\"imag\" + 0.007*\"nasa\" + 0.006*\"graphic\" + 0.006*\"program\" + 0.005*\"launch\" + 0.005*\"file\" + 0.005*\"com\" + 0.005*\"new\" + 0.004*\"orbit\"\n",
- "2019-06-17 00:24:31,627 : INFO : topic #1 (0.200): 0.015*\"com\" + 0.007*\"like\" + 0.007*\"nntp\" + 0.007*\"host\" + 0.006*\"know\" + 0.006*\"univers\" + 0.005*\"henri\" + 0.005*\"work\" + 0.005*\"bit\" + 0.005*\"think\"\n",
- "2019-06-17 00:24:31,628 : INFO : topic #2 (0.200): 0.014*\"armenian\" + 0.011*\"peopl\" + 0.009*\"turkish\" + 0.007*\"jew\" + 0.007*\"said\" + 0.006*\"right\" + 0.005*\"know\" + 0.005*\"kill\" + 0.005*\"isra\" + 0.005*\"turkei\"\n",
- "2019-06-17 00:24:31,629 : INFO : topic #3 (0.200): 0.012*\"com\" + 0.010*\"israel\" + 0.009*\"bike\" + 0.006*\"isra\" + 0.006*\"dod\" + 0.005*\"like\" + 0.005*\"ride\" + 0.005*\"host\" + 0.005*\"nntp\" + 0.005*\"motorcycl\"\n",
- "2019-06-17 00:24:31,631 : INFO : topic #4 (0.200): 0.011*\"god\" + 0.008*\"peopl\" + 0.007*\"think\" + 0.006*\"exist\" + 0.006*\"univers\" + 0.005*\"com\" + 0.005*\"believ\" + 0.005*\"islam\" + 0.005*\"moral\" + 0.005*\"christian\"\n",
- "2019-06-17 00:24:31,631 : INFO : topic diff=0.204980, rho=0.357622\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "2019-06-17 00:24:36,879 : INFO : CorpusAccumulator accumulated stats from 1000 documents\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "/home/misha/envs/gensim/lib/python3.7/site-packages/sklearn/linear_model/logistic.py:758: ConvergenceWarning: lbfgs failed to converge. Increase the number of iterations.\n",
- " \"of iterations.\", ConvergenceWarning)\n",
- "2019-06-17 00:24:46,360 : INFO : CorpusAccumulator accumulated stats from 1000 documents\n",
- "2019-06-17 00:24:46,436 : INFO : running NMF training, 5 topics, 5 passes over the supplied corpus of 2819 documents, evaluating l2 norm every 2819 documents\n",
- "2019-06-17 00:24:46,471 : INFO : PROGRESS: pass 0, at document #1000/2819\n",
- "2019-06-17 00:24:46,497 : INFO : W error diff: -inf\n",
- "2019-06-17 00:24:46,522 : INFO : PROGRESS: pass 0, at document #2000/2819\n",
- "2019-06-17 00:24:46,540 : INFO : W error diff: -2.1783593508632997\n",
- "2019-06-17 00:24:46,561 : INFO : PROGRESS: pass 0, at document #2819/2819\n",
- "2019-06-17 00:24:46,914 : INFO : L2 norm: 28.137070033533682\n",
- "2019-06-17 00:24:46,970 : INFO : topic #0 (0.404): 0.012*\"isra\" + 0.011*\"israel\" + 0.007*\"arab\" + 0.006*\"jew\" + 0.005*\"palestinian\" + 0.004*\"henri\" + 0.004*\"toronto\" + 0.003*\"question\" + 0.003*\"kill\" + 0.003*\"polici\"\n",
- "2019-06-17 00:24:46,972 : INFO : topic #1 (0.358): 0.010*\"space\" + 0.005*\"access\" + 0.005*\"nasa\" + 0.004*\"pat\" + 0.004*\"digex\" + 0.004*\"orbit\" + 0.004*\"shuttl\" + 0.003*\"graphic\" + 0.003*\"data\" + 0.003*\"com\"\n",
- "2019-06-17 00:24:46,974 : INFO : topic #2 (0.388): 0.015*\"armenian\" + 0.007*\"turkish\" + 0.005*\"greek\" + 0.005*\"peopl\" + 0.005*\"armenia\" + 0.005*\"turk\" + 0.005*\"argic\" + 0.004*\"bike\" + 0.004*\"serdar\" + 0.004*\"turkei\"\n",
- "2019-06-17 00:24:46,975 : INFO : topic #3 (0.423): 0.011*\"moral\" + 0.007*\"keith\" + 0.004*\"anim\" + 0.004*\"jake\" + 0.004*\"boni\" + 0.004*\"act\" + 0.004*\"instinct\" + 0.003*\"think\" + 0.003*\"caltech\" + 0.003*\"object\"\n",
- "2019-06-17 00:24:46,977 : INFO : topic #4 (0.441): 0.011*\"islam\" + 0.010*\"god\" + 0.007*\"muslim\" + 0.007*\"livesei\" + 0.007*\"imag\" + 0.006*\"sgi\" + 0.006*\"jaeger\" + 0.005*\"jon\" + 0.005*\"solntz\" + 0.005*\"wpd\"\n",
- "2019-06-17 00:24:46,992 : INFO : W error diff: -0.6087333117616911\n",
- "2019-06-17 00:24:47,031 : INFO : PROGRESS: pass 1, at document #1000/2819\n",
- "2019-06-17 00:24:47,047 : INFO : W error diff: -1.5858439279007879\n",
- "2019-06-17 00:24:47,067 : INFO : PROGRESS: pass 1, at document #2000/2819\n",
- "2019-06-17 00:24:47,078 : INFO : W error diff: -1.1329837530094071\n",
- "2019-06-17 00:24:47,096 : INFO : PROGRESS: pass 1, at document #2819/2819\n",
- "2019-06-17 00:24:47,468 : INFO : L2 norm: 28.02006726219276\n",
- "2019-06-17 00:24:47,539 : INFO : topic #0 (0.345): 0.013*\"israel\" + 0.013*\"isra\" + 0.008*\"arab\" + 0.007*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.003*\"peac\" + 0.003*\"polici\" + 0.003*\"attack\" + 0.003*\"henri\"\n",
- "2019-06-17 00:24:47,542 : INFO : topic #1 (0.253): 0.011*\"space\" + 0.007*\"nasa\" + 0.005*\"access\" + 0.004*\"orbit\" + 0.004*\"pat\" + 0.004*\"digex\" + 0.004*\"launch\" + 0.004*\"shuttl\" + 0.004*\"graphic\" + 0.004*\"com\"\n",
- "2019-06-17 00:24:47,544 : INFO : topic #2 (0.299): 0.020*\"armenian\" + 0.010*\"turkish\" + 0.007*\"armenia\" + 0.006*\"turk\" + 0.006*\"argic\" + 0.006*\"serdar\" + 0.005*\"greek\" + 0.005*\"turkei\" + 0.004*\"genocid\" + 0.004*\"peopl\"\n",
- "2019-06-17 00:24:47,545 : INFO : topic #3 (0.353): 0.013*\"moral\" + 0.011*\"keith\" + 0.006*\"object\" + 0.005*\"caltech\" + 0.005*\"schneider\" + 0.004*\"anim\" + 0.004*\"allan\" + 0.004*\"cco\" + 0.004*\"jake\" + 0.004*\"boni\"\n",
- "2019-06-17 00:24:47,547 : INFO : topic #4 (0.380): 0.013*\"islam\" + 0.013*\"god\" + 0.007*\"livesei\" + 0.007*\"sgi\" + 0.007*\"jaeger\" + 0.006*\"muslim\" + 0.006*\"jon\" + 0.005*\"religion\" + 0.005*\"imag\" + 0.005*\"solntz\"\n"
- ]
- },
- {
- "name": "stderr",
- "output_type": "stream",
- "text": [
- "2019-06-17 00:24:47,556 : INFO : W error diff: -0.05304441334403265\n",
- "2019-06-17 00:24:47,595 : INFO : PROGRESS: pass 2, at document #1000/2819\n",
- "2019-06-17 00:24:47,607 : INFO : W error diff: -0.6532464912217009\n",
- "2019-06-17 00:24:47,629 : INFO : PROGRESS: pass 2, at document #2000/2819\n",
- "2019-06-17 00:24:47,638 : INFO : W error diff: -0.5542774416923812\n",
- "2019-06-17 00:24:47,655 : INFO : PROGRESS: pass 2, at document #2819/2819\n",
- "2019-06-17 00:24:47,973 : INFO : L2 norm: 27.999892226543682\n",
- "2019-06-17 00:24:48,036 : INFO : topic #0 (0.343): 0.013*\"israel\" + 0.013*\"isra\" + 0.008*\"arab\" + 0.007*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.003*\"peac\" + 0.003*\"attack\" + 0.003*\"polici\" + 0.003*\"lebanon\"\n",
- "2019-06-17 00:24:48,038 : INFO : topic #1 (0.229): 0.010*\"space\" + 0.007*\"nasa\" + 0.005*\"access\" + 0.004*\"orbit\" + 0.004*\"pat\" + 0.004*\"launch\" + 0.004*\"digex\" + 0.004*\"gov\" + 0.004*\"graphic\" + 0.004*\"com\"\n",
- "2019-06-17 00:24:48,039 : INFO : topic #2 (0.283): 0.021*\"armenian\" + 0.010*\"turkish\" + 0.007*\"armenia\" + 0.006*\"turk\" + 0.006*\"argic\" + 0.006*\"serdar\" + 0.005*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.004*\"soviet\"\n",
- "2019-06-17 00:24:48,041 : INFO : topic #3 (0.347): 0.014*\"moral\" + 0.013*\"keith\" + 0.006*\"object\" + 0.006*\"caltech\" + 0.005*\"schneider\" + 0.004*\"allan\" + 0.004*\"cco\" + 0.004*\"anim\" + 0.003*\"jake\" + 0.003*\"natur\"\n",
- "2019-06-17 00:24:48,043 : INFO : topic #4 (0.365): 0.014*\"god\" + 0.013*\"islam\" + 0.007*\"livesei\" + 0.007*\"sgi\" + 0.007*\"jaeger\" + 0.006*\"muslim\" + 0.006*\"religion\" + 0.005*\"jon\" + 0.005*\"atheist\" + 0.005*\"atheism\"\n",
- "2019-06-17 00:24:48,050 : INFO : W error diff: 0.06399021760879364\n",
- "2019-06-17 00:24:48,090 : INFO : PROGRESS: pass 3, at document #1000/2819\n",
- "2019-06-17 00:24:48,103 : INFO : W error diff: -0.3678424933365889\n",
- "2019-06-17 00:24:48,123 : INFO : PROGRESS: pass 3, at document #2000/2819\n",
- "2019-06-17 00:24:48,131 : INFO : W error diff: -0.34924666183303543\n",
- "2019-06-17 00:24:48,148 : INFO : PROGRESS: pass 3, at document #2819/2819\n",
- "2019-06-17 00:24:48,460 : INFO : L2 norm: 27.991268049236886\n",
- "2019-06-17 00:24:48,518 : INFO : topic #0 (0.350): 0.014*\"israel\" + 0.013*\"isra\" + 0.008*\"arab\" + 0.007*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"lebanon\" + 0.003*\"polici\"\n",
- "2019-06-17 00:24:48,520 : INFO : topic #1 (0.220): 0.010*\"space\" + 0.007*\"nasa\" + 0.005*\"access\" + 0.004*\"orbit\" + 0.004*\"launch\" + 0.004*\"pat\" + 0.004*\"gov\" + 0.004*\"com\" + 0.004*\"digex\" + 0.004*\"alaska\"\n",
- "2019-06-17 00:24:48,521 : INFO : topic #2 (0.282): 0.021*\"armenian\" + 0.010*\"turkish\" + 0.007*\"armenia\" + 0.007*\"turk\" + 0.006*\"argic\" + 0.006*\"serdar\" + 0.005*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.004*\"soviet\"\n",
- "2019-06-17 00:24:48,523 : INFO : topic #3 (0.351): 0.014*\"moral\" + 0.013*\"keith\" + 0.006*\"object\" + 0.006*\"caltech\" + 0.005*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.003*\"natur\" + 0.003*\"think\"\n",
- "2019-06-17 00:24:48,524 : INFO : topic #4 (0.364): 0.014*\"god\" + 0.013*\"islam\" + 0.007*\"sgi\" + 0.007*\"jaeger\" + 0.007*\"livesei\" + 0.006*\"muslim\" + 0.006*\"religion\" + 0.006*\"atheist\" + 0.005*\"atheism\" + 0.005*\"jon\"\n",
- "2019-06-17 00:24:48,531 : INFO : W error diff: 0.08877110840856872\n",
- "2019-06-17 00:24:48,570 : INFO : PROGRESS: pass 4, at document #1000/2819\n",
- "2019-06-17 00:24:48,584 : INFO : W error diff: -0.2446709705343757\n",
- "2019-06-17 00:24:48,605 : INFO : PROGRESS: pass 4, at document #2000/2819\n",
- "2019-06-17 00:24:48,613 : INFO : W error diff: -0.24931839405260803\n",
- "2019-06-17 00:24:48,635 : INFO : PROGRESS: pass 4, at document #2819/2819\n",
- "2019-06-17 00:24:49,056 : INFO : L2 norm: 27.98648818098989\n",
- "2019-06-17 00:24:49,116 : INFO : topic #0 (0.354): 0.014*\"israel\" + 0.013*\"isra\" + 0.008*\"arab\" + 0.007*\"jew\" + 0.005*\"palestinian\" + 0.004*\"lebanes\" + 0.004*\"peac\" + 0.003*\"attack\" + 0.003*\"lebanon\" + 0.003*\"polici\"\n",
- "2019-06-17 00:24:49,117 : INFO : topic #1 (0.209): 0.010*\"space\" + 0.007*\"nasa\" + 0.005*\"access\" + 0.004*\"orbit\" + 0.004*\"launch\" + 0.004*\"gov\" + 0.004*\"pat\" + 0.004*\"com\" + 0.004*\"alaska\" + 0.004*\"moon\"\n",
- "2019-06-17 00:24:49,119 : INFO : topic #2 (0.283): 0.021*\"armenian\" + 0.010*\"turkish\" + 0.007*\"armenia\" + 0.007*\"argic\" + 0.007*\"turk\" + 0.006*\"serdar\" + 0.005*\"turkei\" + 0.005*\"greek\" + 0.005*\"genocid\" + 0.004*\"soviet\"\n",
- "2019-06-17 00:24:49,120 : INFO : topic #3 (0.356): 0.015*\"moral\" + 0.014*\"keith\" + 0.006*\"object\" + 0.006*\"caltech\" + 0.005*\"schneider\" + 0.005*\"allan\" + 0.005*\"cco\" + 0.004*\"anim\" + 0.003*\"natur\" + 0.003*\"goal\"\n",
- "2019-06-17 00:24:49,122 : INFO : topic #4 (0.366): 0.014*\"god\" + 0.014*\"islam\" + 0.007*\"jaeger\" + 0.006*\"sgi\" + 0.006*\"livesei\" + 0.006*\"muslim\" + 0.006*\"atheist\" + 0.006*\"religion\" + 0.006*\"atheism\" + 0.005*\"rushdi\"\n",
- "2019-06-17 00:24:49,133 : INFO : W error diff: 0.0932956490045207\n",
- "2019-06-17 00:24:54,464 : INFO : CorpusAccumulator accumulated stats from 1000 documents\n"
- ]
- }
- ],
+ "outputs": [],
"source": [
"tm_metrics = pd.DataFrame(columns=['model', 'train_time', 'coherence', 'l2_norm', 'f1', 'topics'])\n",
"\n",
@@ -1175,21 +996,39 @@
" **fixed_params,\n",
" ),\n",
" 'lda',\n",
- " 1,\n",
+ " 0.1,\n",
")\n",
"row.update(get_metrics(\n",
" lda, test_corpus, train_corpus, y_train, y_test,\n",
"))\n",
"tm_metrics = tm_metrics.append(pd.Series(row), ignore_index=True)\n",
"\n",
+ "# LSI metrics\n",
+ "row = {}\n",
+ "row['model'] = 'lsi'\n",
+ "row['train_time'], row['mean_ram'], row['max_ram'], lsi = get_train_time_and_ram(\n",
+ " lambda: LsiModel(\n",
+ " corpus=train_corpus_tfidf,\n",
+ " num_topics=5,\n",
+ " id2word=dictionary,\n",
+ " chunksize=2000,\n",
+ " ),\n",
+ " 'lsi',\n",
+ " 0.1,\n",
+ ")\n",
+ "row.update(get_metrics(\n",
+ " lsi, test_corpus_tfidf, train_corpus_tfidf, y_train, y_test,\n",
+ "))\n",
+ "tm_metrics = tm_metrics.append(pd.Series(row), ignore_index=True)\n",
+ "\n",
"# Sklearn NMF metrics\n",
"row = {}\n",
"row['model'] = 'sklearn_nmf'\n",
- "train_dense_corpus_tfidf = matutils.corpus2dense(train_corpus_tfidf, len(dictionary)).T\n",
+ "train_csc_corpus_tfidf = matutils.corpus2csc(train_corpus_tfidf, len(dictionary)).T\n",
"row['train_time'], row['mean_ram'], row['max_ram'], sklearn_nmf = get_train_time_and_ram(\n",
- " lambda: SklearnNmf(n_components=5, random_state=42).fit(train_dense_corpus_tfidf),\n",
+ " lambda: SklearnNmf(n_components=5, random_state=42).fit(train_csc_corpus_tfidf),\n",
" 'sklearn_nmf',\n",
- " 1,\n",
+ " 0.1,\n",
")\n",
"row.update(get_metrics(\n",
" sklearn_nmf, test_corpus_tfidf, train_corpus_tfidf, y_train, y_test, dictionary,\n",
@@ -1206,7 +1045,7 @@
" **fixed_params\n",
" ),\n",
" 'gensim_nmf',\n",
- " 0.5,\n",
+ " 0.1,\n",
")\n",
"row.update(get_metrics(\n",
" gensim_nmf, test_corpus_tfidf, train_corpus_tfidf, y_train, y_test,\n",
@@ -1224,7 +1063,7 @@
},
{
"cell_type": "code",
- "execution_count": 18,
+ "execution_count": 21,
"metadata": {},
"outputs": [
{
@@ -1261,45 +1100,56 @@
" \n",
" | 0 | \n",
" lda | \n",
- " 00:00:14 | \n",
+ " 00:00:08.862000 | \n",
" -2.1054 | \n",
" - | \n",
" 0.7511 | \n",
- " 273 MB | \n",
- " 273 MB | \n",
+ " 366 MB | \n",
+ " 366 MB | \n",
"
\n",
" \n",
" | 1 | \n",
+ " lsi | \n",
+ " 00:00:00.332000 | \n",
+ " -5.7010 | \n",
+ " 42.4642 | \n",
+ " 0.8587 | \n",
+ " 381 MB | \n",
+ " 379 MB | \n",
+ "
\n",
+ " \n",
+ " | 2 | \n",
" sklearn_nmf | \n",
- " 00:00:04 | \n",
+ " 00:00:00.166000 | \n",
" -3.1835 | \n",
" 42.4759 | \n",
- " 0.7905 | \n",
- " 538 MB | \n",
- " 538 MB | \n",
+ " 0.7889 | \n",
+ " 378 MB | \n",
+ " 378 MB | \n",
"
\n",
" \n",
- " | 2 | \n",
+ " 3 | \n",
" gensim_nmf | \n",
- " 00:00:03 | \n",
- " -4.0459 | \n",
- " 42.5486 | \n",
- " 0.8044 | \n",
- " 406 MB | \n",
- " 406 MB | \n",
+ " 00:00:00.954000 | \n",
+ " -4.1310 | \n",
+ " 42.5487 | \n",
+ " 0.8065 | \n",
+ " 379 MB | \n",
+ " 379 MB | \n",
"
\n",
" \n",
"\n",
"