Ravin Kohli: FIX weighted loss issue (#94)

refactor_development/_sources/examples/example_image_classification.rst.txt

@@ -80,10 +80,10 @@ Image Classification
       image_augmenter:GaussianBlur:use_augmenter, Value: False
       image_augmenter:GaussianNoise:use_augmenter, Value: False
       image_augmenter:RandomAffine:use_augmenter, Value: False
-      image_augmenter:RandomCutout:p, Value: 0.8490799303808481
+      image_augmenter:RandomCutout:p, Value: 0.5586693024569416
       image_augmenter:RandomCutout:use_augmenter, Value: True
       image_augmenter:Resize:use_augmenter, Value: False
-      image_augmenter:ZeroPadAndCrop:percent, Value: 0.2993076189415605
+      image_augmenter:ZeroPadAndCrop:percent, Value: 0.4581846771624755
       normalizer:__choice__, Value: 'ImageNormalizer'
 
     Fitting the pipeline...
@@ -163,7 +163,7 @@ Image Classification
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 0 minutes  7.334 seconds)
+   **Total running time of the script:** ( 0 minutes  5.798 seconds)
 
 
 .. _sphx_glr_download_examples_example_image_classification.py:

refactor_development/_sources/examples/example_tabular_classification.rst.txt

@@ -36,7 +36,7 @@ with AutoPyTorch
 
  .. code-block:: none
 
-    <smac.runhistory.runhistory.RunHistory object at 0x7fd25a7cde50> [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
+    <smac.runhistory.runhistory.RunHistory object at 0x7f3fac518ee0> [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
       data_loader:batch_size, Value: 32
       encoder:__choice__, Value: 'OneHotEncoder'
       feature_preprocessor:__choice__, Value: 'NoFeaturePreprocessor'
@@ -67,7 +67,7 @@ with AutoPyTorch
       scaler:__choice__, Value: 'StandardScaler'
       trainer:StandardTrainer:weighted_loss, Value: True
       trainer:__choice__, Value: 'StandardTrainer'
-    , ta_runs=0, ta_time_used=0.0, wallclock_time=0.0015573501586914062, budget=0), TrajEntry(train_perf=0.17543859649122806, incumbent_id=1, incumbent=Configuration:
+    , ta_runs=0, ta_time_used=0.0, wallclock_time=0.001955747604370117, budget=0), TrajEntry(train_perf=0.1578947368421053, incumbent_id=1, incumbent=Configuration:
       data_loader:batch_size, Value: 32
       encoder:__choice__, Value: 'OneHotEncoder'
       feature_preprocessor:__choice__, Value: 'NoFeaturePreprocessor'
@@ -98,8 +98,43 @@ with AutoPyTorch
       scaler:__choice__, Value: 'StandardScaler'
       trainer:StandardTrainer:weighted_loss, Value: True
       trainer:__choice__, Value: 'StandardTrainer'
-    , ta_runs=1, ta_time_used=3.7913620471954346, wallclock_time=5.133898019790649, budget=5.555555555555555)]
-    {'accuracy': 0.8728323699421965}
+    , ta_runs=1, ta_time_used=4.385937213897705, wallclock_time=5.783432245254517, budget=5.555555555555555), TrajEntry(train_perf=0.15204678362573099, incumbent_id=2, incumbent=Configuration:
+      data_loader:batch_size, Value: 426
+      encoder:__choice__, Value: 'OrdinalEncoder'
+      feature_preprocessor:Nystroem:coef0, Value: -0.4848435864342966
+      feature_preprocessor:Nystroem:kernel, Value: 'sigmoid'
+      feature_preprocessor:Nystroem:n_components, Value: 4
+      feature_preprocessor:__choice__, Value: 'Nystroem'
+      imputer:categorical_strategy, Value: 'constant_!missing!'
+      imputer:numerical_strategy, Value: 'most_frequent'
+      lr_scheduler:CosineAnnealingLR:T_max, Value: 56
+      lr_scheduler:__choice__, Value: 'CosineAnnealingLR'
+      network_backbone:ShapedMLPBackbone:activation, Value: 'tanh'
+      network_backbone:ShapedMLPBackbone:max_units, Value: 999
+      network_backbone:ShapedMLPBackbone:mlp_shape, Value: 'triangle'
+      network_backbone:ShapedMLPBackbone:num_groups, Value: 11
+      network_backbone:ShapedMLPBackbone:output_dim, Value: 477
+      network_backbone:ShapedMLPBackbone:use_dropout, Value: False
+      network_backbone:__choice__, Value: 'ShapedMLPBackbone'
+      network_head:__choice__, Value: 'fully_connected'
+      network_head:fully_connected:activation, Value: 'tanh'
+      network_head:fully_connected:num_layers, Value: 4
+      network_head:fully_connected:units_layer_1, Value: 189
+      network_head:fully_connected:units_layer_2, Value: 259
+      network_head:fully_connected:units_layer_3, Value: 494
+      network_init:OrthogonalInit:bias_strategy, Value: 'Zero'
+      network_init:__choice__, Value: 'OrthogonalInit'
+      optimizer:AdamOptimizer:beta1, Value: 0.8999825268789966
+      optimizer:AdamOptimizer:beta2, Value: 0.9200091936462466
+      optimizer:AdamOptimizer:lr, Value: 0.0006438744148679775
+      optimizer:AdamOptimizer:weight_decay, Value: 0.03262472357115608
+      optimizer:__choice__, Value: 'AdamOptimizer'
+      scaler:__choice__, Value: 'NoScaler'
+      trainer:MixUpTrainer:alpha, Value: 0.08759596707798334
+      trainer:MixUpTrainer:weighted_loss, Value: False
+      trainer:__choice__, Value: 'MixUpTrainer'
+    , ta_runs=33, ta_time_used=385.77361822128296, wallclock_time=484.24447774887085, budget=50.0)]
+    {'accuracy': 0.861271676300578}
 
 
 
@@ -190,7 +225,7 @@ with AutoPyTorch
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 8 minutes  58.645 seconds)
+   **Total running time of the script:** ( 9 minutes  13.196 seconds)
 
 
 .. _sphx_glr_download_examples_example_tabular_classification.py:

refactor_development/_sources/examples/example_tabular_regression.rst.txt

@@ -36,7 +36,7 @@ with AutoPyTorch
 
  .. code-block:: none
 
-    <smac.runhistory.runhistory.RunHistory object at 0x7fd258d6b850> [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
+    <smac.runhistory.runhistory.RunHistory object at 0x7f3f94e1f760> [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
       data_loader:batch_size, Value: 32
       encoder:__choice__, Value: 'NoEncoder'
       imputer:numerical_strategy, Value: 'mean'
@@ -64,7 +64,7 @@ with AutoPyTorch
       optimizer:__choice__, Value: 'AdamOptimizer'
       scaler:__choice__, Value: 'StandardScaler'
       trainer:__choice__, Value: 'StandardTrainer'
-    , ta_runs=0, ta_time_used=0.0, wallclock_time=0.0016045570373535156, budget=0), TrajEntry(train_perf=0.00022481081457881302, incumbent_id=1, incumbent=Configuration:
+    , ta_runs=0, ta_time_used=0.0, wallclock_time=0.0017769336700439453, budget=0), TrajEntry(train_perf=0.00043087196655655635, incumbent_id=1, incumbent=Configuration:
       data_loader:batch_size, Value: 32
       encoder:__choice__, Value: 'NoEncoder'
       imputer:numerical_strategy, Value: 'mean'
@@ -92,8 +92,8 @@ with AutoPyTorch
       optimizer:__choice__, Value: 'AdamOptimizer'
       scaler:__choice__, Value: 'StandardScaler'
       trainer:__choice__, Value: 'StandardTrainer'
-    , ta_runs=1, ta_time_used=6.372622489929199, wallclock_time=9.338410377502441, budget=5.555555555555555)]
-    {'r2': 0.9998769993075737}
+    , ta_runs=1, ta_time_used=7.4215779304504395, wallclock_time=10.76639461517334, budget=5.555555555555555)]
+    {'r2': 0.9998200109852567}
 
 
 
@@ -216,7 +216,7 @@ with AutoPyTorch
 
 .. rst-class:: sphx-glr-timing
 
-   **Total running time of the script:** ( 8 minutes  27.763 seconds)
+   **Total running time of the script:** ( 8 minutes  26.163 seconds)
 
 
 .. _sphx_glr_download_examples_example_tabular_regression.py:

refactor_development/_sources/examples/sg_execution_times.rst.txt

@@ -5,12 +5,12 @@
 
 Computation times
 =================
-**17:33.741** total execution time for **examples** files:
+**17:45.156** total execution time for **examples** files:
 
 +----------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_example_tabular_classification.py` (``example_tabular_classification.py``) | 08:58.645 | 0.0 MB |
+| :ref:`sphx_glr_examples_example_tabular_classification.py` (``example_tabular_classification.py``) | 09:13.196 | 0.0 MB |
 +----------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_example_tabular_regression.py` (``example_tabular_regression.py``)         | 08:27.763 | 0.0 MB |
+| :ref:`sphx_glr_examples_example_tabular_regression.py` (``example_tabular_regression.py``)         | 08:26.163 | 0.0 MB |
 +----------------------------------------------------------------------------------------------------+-----------+--------+
-| :ref:`sphx_glr_examples_example_image_classification.py` (``example_image_classification.py``)     | 00:07.334 | 0.0 MB |
+| :ref:`sphx_glr_examples_example_image_classification.py` (``example_image_classification.py``)     | 00:05.798 | 0.0 MB |
 +----------------------------------------------------------------------------------------------------+-----------+--------+

refactor_development/examples/example_image_classification.html

@@ -3942,10 +3942,10 @@
   image_augmenter:GaussianBlur:use_augmenter, Value: False
   image_augmenter:GaussianNoise:use_augmenter, Value: False
   image_augmenter:RandomAffine:use_augmenter, Value: False
-  image_augmenter:RandomCutout:p, Value: 0.8490799303808481
+  image_augmenter:RandomCutout:p, Value: 0.5586693024569416
   image_augmenter:RandomCutout:use_augmenter, Value: True
   image_augmenter:Resize:use_augmenter, Value: False
-  image_augmenter:ZeroPadAndCrop:percent, Value: 0.2993076189415605
+  image_augmenter:ZeroPadAndCrop:percent, Value: 0.4581846771624755
   normalizer:__choice__, Value: 'ImageNormalizer'
 
 Fitting the pipeline...
@@ -4018,7 +4018,7 @@
 <span class="nb">print</span><span class="p">(</span><span class="n">pipeline</span><span class="p">)</span>
 </pre></div>
 </div>
-<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 0 minutes  7.334 seconds)</p>
+<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 0 minutes  5.798 seconds)</p>
 <div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-examples-example-image-classification-py">
 <div class="binder-badge docutils container">
 <a class="reference external image-reference" href="https://mybinder.org/v2/gh/automl/Auto-PyTorch/refactor_development?urlpath=lab/tree/notebooks/examples/example_image_classification.ipynb"><img alt="Launch binder" src="../_images/binder_badge_logo.svg" width="150px" /></a>

refactor_development/examples/example_tabular_classification.html

@@ -122,7 +122,7 @@
 <p>The following example shows how to fit a sample classification model
 with AutoPyTorch</p>
 <p class="sphx-glr-script-out">Out:</p>
-<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>&lt;smac.runhistory.runhistory.RunHistory object at 0x7fd25a7cde50&gt; [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
+<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>&lt;smac.runhistory.runhistory.RunHistory object at 0x7f3fac518ee0&gt; [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
   data_loader:batch_size, Value: 32
   encoder:__choice__, Value: &#39;OneHotEncoder&#39;
   feature_preprocessor:__choice__, Value: &#39;NoFeaturePreprocessor&#39;
@@ -153,7 +153,7 @@
   scaler:__choice__, Value: &#39;StandardScaler&#39;
   trainer:StandardTrainer:weighted_loss, Value: True
   trainer:__choice__, Value: &#39;StandardTrainer&#39;
-, ta_runs=0, ta_time_used=0.0, wallclock_time=0.0015573501586914062, budget=0), TrajEntry(train_perf=0.17543859649122806, incumbent_id=1, incumbent=Configuration:
+, ta_runs=0, ta_time_used=0.0, wallclock_time=0.001955747604370117, budget=0), TrajEntry(train_perf=0.1578947368421053, incumbent_id=1, incumbent=Configuration:
   data_loader:batch_size, Value: 32
   encoder:__choice__, Value: &#39;OneHotEncoder&#39;
   feature_preprocessor:__choice__, Value: &#39;NoFeaturePreprocessor&#39;
@@ -184,8 +184,43 @@
   scaler:__choice__, Value: &#39;StandardScaler&#39;
   trainer:StandardTrainer:weighted_loss, Value: True
   trainer:__choice__, Value: &#39;StandardTrainer&#39;
-, ta_runs=1, ta_time_used=3.7913620471954346, wallclock_time=5.133898019790649, budget=5.555555555555555)]
-{&#39;accuracy&#39;: 0.8728323699421965}
+, ta_runs=1, ta_time_used=4.385937213897705, wallclock_time=5.783432245254517, budget=5.555555555555555), TrajEntry(train_perf=0.15204678362573099, incumbent_id=2, incumbent=Configuration:
+  data_loader:batch_size, Value: 426
+  encoder:__choice__, Value: &#39;OrdinalEncoder&#39;
+  feature_preprocessor:Nystroem:coef0, Value: -0.4848435864342966
+  feature_preprocessor:Nystroem:kernel, Value: &#39;sigmoid&#39;
+  feature_preprocessor:Nystroem:n_components, Value: 4
+  feature_preprocessor:__choice__, Value: &#39;Nystroem&#39;
+  imputer:categorical_strategy, Value: &#39;constant_!missing!&#39;
+  imputer:numerical_strategy, Value: &#39;most_frequent&#39;
+  lr_scheduler:CosineAnnealingLR:T_max, Value: 56
+  lr_scheduler:__choice__, Value: &#39;CosineAnnealingLR&#39;
+  network_backbone:ShapedMLPBackbone:activation, Value: &#39;tanh&#39;
+  network_backbone:ShapedMLPBackbone:max_units, Value: 999
+  network_backbone:ShapedMLPBackbone:mlp_shape, Value: &#39;triangle&#39;
+  network_backbone:ShapedMLPBackbone:num_groups, Value: 11
+  network_backbone:ShapedMLPBackbone:output_dim, Value: 477
+  network_backbone:ShapedMLPBackbone:use_dropout, Value: False
+  network_backbone:__choice__, Value: &#39;ShapedMLPBackbone&#39;
+  network_head:__choice__, Value: &#39;fully_connected&#39;
+  network_head:fully_connected:activation, Value: &#39;tanh&#39;
+  network_head:fully_connected:num_layers, Value: 4
+  network_head:fully_connected:units_layer_1, Value: 189
+  network_head:fully_connected:units_layer_2, Value: 259
+  network_head:fully_connected:units_layer_3, Value: 494
+  network_init:OrthogonalInit:bias_strategy, Value: &#39;Zero&#39;
+  network_init:__choice__, Value: &#39;OrthogonalInit&#39;
+  optimizer:AdamOptimizer:beta1, Value: 0.8999825268789966
+  optimizer:AdamOptimizer:beta2, Value: 0.9200091936462466
+  optimizer:AdamOptimizer:lr, Value: 0.0006438744148679775
+  optimizer:AdamOptimizer:weight_decay, Value: 0.03262472357115608
+  optimizer:__choice__, Value: &#39;AdamOptimizer&#39;
+  scaler:__choice__, Value: &#39;NoScaler&#39;
+  trainer:MixUpTrainer:alpha, Value: 0.08759596707798334
+  trainer:MixUpTrainer:weighted_loss, Value: False
+  trainer:__choice__, Value: &#39;MixUpTrainer&#39;
+, ta_runs=33, ta_time_used=385.77361822128296, wallclock_time=484.24447774887085, budget=50.0)]
+{&#39;accuracy&#39;: 0.861271676300578}
 </pre></div>
 </div>
 <div class="line-block">
@@ -269,7 +304,7 @@
     <span class="nb">print</span><span class="p">(</span><span class="n">score</span><span class="p">)</span>
 </pre></div>
 </div>
-<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 8 minutes  58.645 seconds)</p>
+<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 9 minutes  13.196 seconds)</p>
 <div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-examples-example-tabular-classification-py">
 <div class="binder-badge docutils container">
 <a class="reference external image-reference" href="https://mybinder.org/v2/gh/automl/Auto-PyTorch/refactor_development?urlpath=lab/tree/notebooks/examples/example_tabular_classification.ipynb"><img alt="Launch binder" src="../_images/binder_badge_logo.svg" width="150px" /></a>

refactor_development/examples/example_tabular_regression.html

@@ -122,7 +122,7 @@
 <p>The following example shows how to fit a sample classification model
 with AutoPyTorch</p>
 <p class="sphx-glr-script-out">Out:</p>
-<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>&lt;smac.runhistory.runhistory.RunHistory object at 0x7fd258d6b850&gt; [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
+<div class="sphx-glr-script-out highlight-none notranslate"><div class="highlight"><pre><span></span>&lt;smac.runhistory.runhistory.RunHistory object at 0x7f3f94e1f760&gt; [TrajEntry(train_perf=2147483648, incumbent_id=1, incumbent=Configuration:
   data_loader:batch_size, Value: 32
   encoder:__choice__, Value: &#39;NoEncoder&#39;
   imputer:numerical_strategy, Value: &#39;mean&#39;
@@ -150,7 +150,7 @@
   optimizer:__choice__, Value: &#39;AdamOptimizer&#39;
   scaler:__choice__, Value: &#39;StandardScaler&#39;
   trainer:__choice__, Value: &#39;StandardTrainer&#39;
-, ta_runs=0, ta_time_used=0.0, wallclock_time=0.0016045570373535156, budget=0), TrajEntry(train_perf=0.00022481081457881302, incumbent_id=1, incumbent=Configuration:
+, ta_runs=0, ta_time_used=0.0, wallclock_time=0.0017769336700439453, budget=0), TrajEntry(train_perf=0.00043087196655655635, incumbent_id=1, incumbent=Configuration:
   data_loader:batch_size, Value: 32
   encoder:__choice__, Value: &#39;NoEncoder&#39;
   imputer:numerical_strategy, Value: &#39;mean&#39;
@@ -178,8 +178,8 @@
   optimizer:__choice__, Value: &#39;AdamOptimizer&#39;
   scaler:__choice__, Value: &#39;StandardScaler&#39;
   trainer:__choice__, Value: &#39;StandardTrainer&#39;
-, ta_runs=1, ta_time_used=6.372622489929199, wallclock_time=9.338410377502441, budget=5.555555555555555)]
-{&#39;r2&#39;: 0.9998769993075737}
+, ta_runs=1, ta_time_used=7.4215779304504395, wallclock_time=10.76639461517334, budget=5.555555555555555)]
+{&#39;r2&#39;: 0.9998200109852567}
 </pre></div>
 </div>
 <div class="line-block">
@@ -295,7 +295,7 @@
     <span class="nb">print</span><span class="p">(</span><span class="n">score</span><span class="p">)</span>
 </pre></div>
 </div>
-<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 8 minutes  27.763 seconds)</p>
+<p class="sphx-glr-timing"><strong>Total running time of the script:</strong> ( 8 minutes  26.163 seconds)</p>
 <div class="sphx-glr-footer class sphx-glr-footer-example docutils container" id="sphx-glr-download-examples-example-tabular-regression-py">
 <div class="binder-badge docutils container">
 <a class="reference external image-reference" href="https://mybinder.org/v2/gh/automl/Auto-PyTorch/refactor_development?urlpath=lab/tree/notebooks/examples/example_tabular_regression.ipynb"><img alt="Launch binder" src="../_images/binder_badge_logo.svg" width="150px" /></a>

refactor_development/examples/sg_execution_times.html

@@ -114,7 +114,7 @@
 
   <div class="section" id="computation-times">
 <span id="sphx-glr-examples-sg-execution-times"></span><h1>Computation times<a class="headerlink" href="#computation-times" title="Permalink to this headline">¶</a></h1>
-<p><strong>17:33.741</strong> total execution time for <strong>examples</strong> files:</p>
+<p><strong>17:45.156</strong> total execution time for <strong>examples</strong> files:</p>
 <table class="docutils align-default">
 <colgroup>
 <col style="width: 84%" />
@@ -123,15 +123,15 @@
 </colgroup>
 <tbody>
 <tr class="row-odd"><td><p><a class="reference internal" href="example_tabular_classification.html#sphx-glr-examples-example-tabular-classification-py"><span class="std std-ref">Tabular Classification</span></a> (<code class="docutils literal notranslate"><span class="pre">example_tabular_classification.py</span></code>)</p></td>
-<td><p>08:58.645</p></td>
+<td><p>09:13.196</p></td>
 <td><p>0.0 MB</p></td>
 </tr>
 <tr class="row-even"><td><p><a class="reference internal" href="example_tabular_regression.html#sphx-glr-examples-example-tabular-regression-py"><span class="std std-ref">Tabular Regression</span></a> (<code class="docutils literal notranslate"><span class="pre">example_tabular_regression.py</span></code>)</p></td>
-<td><p>08:27.763</p></td>
+<td><p>08:26.163</p></td>
 <td><p>0.0 MB</p></td>
 </tr>
 <tr class="row-odd"><td><p><a class="reference internal" href="example_image_classification.html#sphx-glr-examples-example-image-classification-py"><span class="std std-ref">Image Classification</span></a> (<code class="docutils literal notranslate"><span class="pre">example_image_classification.py</span></code>)</p></td>
-<td><p>00:07.334</p></td>
+<td><p>00:05.798</p></td>
 <td><p>0.0 MB</p></td>
 </tr>
 </tbody>