Generated gh-pages for commit 0cd61ac

Author: William H.P. Nielsen <whpn@mailbox.org>

    add debugging and make the final write more robust (#822)

_modules/qcodes/data/data_set.html

@@ -176,6 +176,8 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
 <span class="kn">from</span> <span class="nn">.location</span> <span class="k">import</span> <span class="n">FormatLocation</span>
 <span class="kn">from</span> <span class="nn">qcodes.utils.helpers</span> <span class="k">import</span> <span class="n">DelegateAttributes</span><span class="p">,</span> <span class="n">full_class</span><span class="p">,</span> <span class="n">deep_update</span>
 
+<span class="n">log</span> <span class="o">=</span> <span class="n">logging</span><span class="o">.</span><span class="n">getLogger</span><span class="p">(</span><span class="vm">__name__</span><span class="p">)</span>
+
 
 <div class="viewcode-block" id="new_data"><a class="viewcode-back" href="../../../api/generated/qcodes.new_data.html#qcodes.new_data">[docs]</a><span class="k">def</span> <span class="nf">new_data</span><span class="p">(</span><span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">loc_record</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">name</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">overwrite</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span>
              <span class="n">io</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="o">**</span><span class="n">kwargs</span><span class="p">):</span>
@@ -411,14 +413,14 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
 <span class="sd">        Args:</span>
 <span class="sd">            delay (float): seconds between iterations. Default 1.5</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
+        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span>
             <span class="s1">&#39;waiting for DataSet &lt;</span><span class="si">{}</span><span class="s1">&gt; to complete&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">))</span>
 
         <span class="n">failing</span> <span class="o">=</span> <span class="p">{</span><span class="n">key</span><span class="p">:</span> <span class="kc">False</span> <span class="k">for</span> <span class="n">key</span> <span class="ow">in</span> <span class="bp">self</span><span class="o">.</span><span class="n">background_functions</span><span class="p">}</span>
 
         <span class="n">completed</span> <span class="o">=</span> <span class="kc">False</span>
         <span class="k">while</span> <span class="kc">True</span><span class="p">:</span>
-            <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;DataSet: </span><span class="si">{:.0f}% c</span><span class="s1">omplete&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
+            <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;DataSet: </span><span class="si">{:.0f}% c</span><span class="s1">omplete&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span>
                 <span class="bp">self</span><span class="o">.</span><span class="n">fraction_complete</span><span class="p">()</span> <span class="o">*</span> <span class="mi">100</span><span class="p">))</span>
 
             <span class="c1"># first check if we&#39;re done</span>
@@ -429,13 +431,13 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
             <span class="c1"># because we want things like live plotting to get the final data</span>
             <span class="k">for</span> <span class="n">key</span><span class="p">,</span> <span class="n">fn</span> <span class="ow">in</span> <span class="nb">list</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">background_functions</span><span class="o">.</span><span class="n">items</span><span class="p">()):</span>
                 <span class="k">try</span><span class="p">:</span>
-                    <span class="n">logging</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;calling </span><span class="si">{}</span><span class="s1">: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">repr</span><span class="p">(</span><span class="n">fn</span><span class="p">)))</span>
+                    <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;calling </span><span class="si">{}</span><span class="s1">: </span><span class="si">{}</span><span class="s1">&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">,</span> <span class="nb">repr</span><span class="p">(</span><span class="n">fn</span><span class="p">)))</span>
                     <span class="n">fn</span><span class="p">()</span>
                     <span class="n">failing</span><span class="p">[</span><span class="n">key</span><span class="p">]</span> <span class="o">=</span> <span class="kc">False</span>
                 <span class="k">except</span> <span class="ne">Exception</span><span class="p">:</span>
-                    <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">format_exc</span><span class="p">())</span>
+                    <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="n">format_exc</span><span class="p">())</span>
                     <span class="k">if</span> <span class="n">failing</span><span class="p">[</span><span class="n">key</span><span class="p">]:</span>
-                        <span class="n">logging</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
+                        <span class="n">log</span><span class="o">.</span><span class="n">warning</span><span class="p">(</span>
                             <span class="s1">&#39;background function </span><span class="si">{}</span><span class="s1"> failed twice in a row, &#39;</span>
                             <span class="s1">&#39;removing it&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="n">key</span><span class="p">))</span>
                         <span class="k">del</span> <span class="bp">self</span><span class="o">.</span><span class="n">background_functions</span><span class="p">[</span><span class="n">key</span><span class="p">]</span>
@@ -447,7 +449,7 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
             <span class="c1"># but only sleep if we&#39;re not already finished</span>
             <span class="n">time</span><span class="o">.</span><span class="n">sleep</span><span class="p">(</span><span class="n">delay</span><span class="p">)</span>
 
-        <span class="n">logging</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;DataSet &lt;</span><span class="si">{}</span><span class="s1">&gt; is complete&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">))</span>
+        <span class="n">log</span><span class="o">.</span><span class="n">info</span><span class="p">(</span><span class="s1">&#39;DataSet &lt;</span><span class="si">{}</span><span class="s1">&gt; is complete&#39;</span><span class="o">.</span><span class="n">format</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">))</span>
 
     <span class="k">def</span> <span class="nf">get_changes</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">synced_indices</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
@@ -552,8 +554,11 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
         <span class="bp">self</span><span class="o">.</span><span class="n">last_store</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
         <span class="k">if</span> <span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">write_period</span> <span class="ow">is</span> <span class="ow">not</span> <span class="kc">None</span> <span class="ow">and</span>
                 <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span> <span class="o">&gt;</span> <span class="bp">self</span><span class="o">.</span><span class="n">last_write</span> <span class="o">+</span> <span class="bp">self</span><span class="o">.</span><span class="n">write_period</span><span class="p">):</span>
+            <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Attempting to write&#39;</span><span class="p">)</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">()</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">last_write</span> <span class="o">=</span> <span class="n">time</span><span class="o">.</span><span class="n">time</span><span class="p">()</span>
+        <span class="k">else</span><span class="p">:</span>
+            <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;.store method: This is not the right time to write&#39;</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">default_parameter_name</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">paramname</span><span class="o">=</span><span class="s1">&#39;amplitude&#39;</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot; Return name of default parameter for plotting</span>
@@ -629,22 +634,26 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
             <span class="k">return</span>
         <span class="bp">self</span><span class="o">.</span><span class="n">formatter</span><span class="o">.</span><span class="n">read_metadata</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>
 
-    <span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">write_metadata</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+    <span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">write_metadata</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">only_complete</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Writes updates to the DataSet to storage.</span>
 <span class="sd">        N.B. it is recommended to call data_set.finalize() when a DataSet is</span>
 <span class="sd">        no longer expected to change to ensure files get closed</span>
 
 <span class="sd">        Args:</span>
 <span class="sd">            write_metadata (bool): write the metadata to disk</span>
+<span class="sd">            only_complete (bool): passed on to the match_save_range inside</span>
+<span class="sd">                self.formatter.write. Used to ensure that all new data gets</span>
+<span class="sd">                saved even when some columns are strange.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">if</span> <span class="bp">self</span><span class="o">.</span><span class="n">location</span> <span class="ow">is</span> <span class="kc">False</span><span class="p">:</span>
             <span class="k">return</span>
 
         <span class="bp">self</span><span class="o">.</span><span class="n">formatter</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span>
                              <span class="bp">self</span><span class="o">.</span><span class="n">io</span><span class="p">,</span>
                              <span class="bp">self</span><span class="o">.</span><span class="n">location</span><span class="p">,</span>
-                             <span class="n">write_metadata</span><span class="o">=</span><span class="n">write_metadata</span><span class="p">)</span>
+                             <span class="n">write_metadata</span><span class="o">=</span><span class="n">write_metadata</span><span class="p">,</span>
+                             <span class="n">only_complete</span><span class="o">=</span><span class="n">only_complete</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">write_copy</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">path</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">io_manager</span><span class="o">=</span><span class="kc">None</span><span class="p">,</span> <span class="n">location</span><span class="o">=</span><span class="kc">None</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
@@ -725,7 +734,9 @@ <h1>Source code for qcodes.data.data_set</h1><div class="highlight"><pre>
 <span class="sd">        Also closes the data file(s), if the ``Formatter`` we&#39;re using</span>
 <span class="sd">        supports that.</span>
 <span class="sd">        &quot;&quot;&quot;</span>
-        <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">()</span>
+        <span class="n">log</span><span class="o">.</span><span class="n">debug</span><span class="p">(</span><span class="s1">&#39;Finalising the DataSet. Writing.&#39;</span><span class="p">)</span>
+        <span class="c1"># write all new data, not only (to?) complete columns</span>
+        <span class="bp">self</span><span class="o">.</span><span class="n">write</span><span class="p">(</span><span class="n">only_complete</span><span class="o">=</span><span class="kc">False</span><span class="p">)</span>
 
         <span class="k">if</span> <span class="nb">hasattr</span><span class="p">(</span><span class="bp">self</span><span class="o">.</span><span class="n">formatter</span><span class="p">,</span> <span class="s1">&#39;close_file&#39;</span><span class="p">):</span>
             <span class="bp">self</span><span class="o">.</span><span class="n">formatter</span><span class="o">.</span><span class="n">close_file</span><span class="p">(</span><span class="bp">self</span><span class="p">)</span>

_modules/qcodes/data/format.html

@@ -213,7 +213,7 @@ <h1>Source code for qcodes.data.format</h1><div class="highlight"><pre>
     <span class="n">ArrayGroup</span> <span class="o">=</span> <span class="n">namedtuple</span><span class="p">(</span><span class="s1">&#39;ArrayGroup&#39;</span><span class="p">,</span> <span class="s1">&#39;shape set_arrays data name&#39;</span><span class="p">)</span>
 
     <span class="k">def</span> <span class="nf">write</span><span class="p">(</span><span class="bp">self</span><span class="p">,</span> <span class="n">data_set</span><span class="p">,</span> <span class="n">io_manager</span><span class="p">,</span> <span class="n">location</span><span class="p">,</span> <span class="n">write_metadata</span><span class="o">=</span><span class="kc">True</span><span class="p">,</span>
-              <span class="n">force_write</span><span class="o">=</span><span class="kc">False</span><span class="p">):</span>
+              <span class="n">force_write</span><span class="o">=</span><span class="kc">False</span><span class="p">,</span> <span class="n">only_complete</span><span class="o">=</span><span class="kc">True</span><span class="p">):</span>
         <span class="sd">&quot;&quot;&quot;</span>
 <span class="sd">        Write the DataSet to storage.</span>
 
@@ -228,6 +228,8 @@ <h1>Source code for qcodes.data.format</h1><div class="highlight"><pre>
 <span class="sd">            location (str): the file location within the io_manager.</span>
 <span class="sd">            write_metadata (bool): if True, then the metadata is written to disk</span>
 <span class="sd">            force_write (bool): if True, then the data is written to disk</span>
+<span class="sd">            only_complete (bool): Used only by the gnuplot formatter&#39;s</span>
+<span class="sd">                overridden version of this method</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="k">raise</span> <span class="ne">NotImplementedError</span>
 
@@ -355,7 +357,9 @@ <h1>Source code for qcodes.data.format</h1><div class="highlight"><pre>
 
 <span class="sd">        Returns:</span>
 <span class="sd">            Tuple(int, int): the first and last raveled indices that should</span>
-<span class="sd">                be saved.</span>
+<span class="sd">                be saved. Returns None if:</span>
+<span class="sd">                    * no data is present</span>
+<span class="sd">                    * no new data can be found</span>
 <span class="sd">        &quot;&quot;&quot;</span>
         <span class="n">inner_setpoint</span> <span class="o">=</span> <span class="n">group</span><span class="o">.</span><span class="n">set_arrays</span><span class="p">[</span><span class="o">-</span><span class="mi">1</span><span class="p">]</span>
         <span class="n">full_dim_data</span> <span class="o">=</span> <span class="p">(</span><span class="n">inner_setpoint</span><span class="p">,</span> <span class="p">)</span> <span class="o">+</span> <span class="n">group</span><span class="o">.</span><span class="n">data</span>