Merge pull request #199 from Microsoft/XForm.JuneSS

Handle table column files not present and re-build table.

XForm/XForm/HttpService.cs

@@ -221,6 +221,13 @@ private void Run(string query, string format, int rowCountLimit, int colCountLim
                 // Run the query and return the output
                 pipeline.RunWithoutDispose();
             }
+            catch(ColumnDataNotFoundException ex)
+            {
+                // If column data is missing, delete the table to try to spur re-creating it
+                // NOTE: This logic will likely need to be updated when columns are downloaded remotely; multi-threaded scenarios will be complex.
+                string tablePath = Path.Combine(ex.ColumnPath, @"..\..\..");
+                TableMetadataSerializer.Delete(_xDatabaseContext.StreamProvider, tablePath);
+            }
             finally
             {
                 if (pipeline != null)

XForm/XForm/IO/ColumnDataNotFoundException.cs

@@ -0,0 +1,22 @@
+using System;
+using System.IO;
+using System.Runtime.Serialization;
+
+namespace XForm.IO
+{
+    [Serializable]
+    public class ColumnDataNotFoundException : IOException
+    {
+        public string ColumnPath { get; set; }
+
+        public ColumnDataNotFoundException() { }
+        public ColumnDataNotFoundException(string message) : base(message) { }
+        public ColumnDataNotFoundException(string message, Exception inner) : base(message, inner) { }
+        protected ColumnDataNotFoundException(SerializationInfo info, StreamingContext context) : base(info, context) { }
+
+        public ColumnDataNotFoundException(string message, string columnPath) : base(message)
+        {
+            ColumnPath = columnPath;
+        }
+    }
+}

XForm/XForm/IO/EnumReaderWriter.cs

@@ -4,8 +4,9 @@
 using System;
 using System.Collections.Generic;
 using System.IO;
-
+using System.Runtime.Serialization;
 using XForm.Data;
+using XForm.Extensions;
 using XForm.IO.StreamProvider;
 using XForm.Types;
 
@@ -334,12 +335,11 @@ private EnumReader(IColumnReader valueReader, IColumnReader rowIndexReader)
 
         public static IColumnReader Wrap(IStreamProvider streamProvider, Type columnType, string columnPath, CachingOption option)
         {
-            // Build a reader for the row indices (will be null if the 'VR.u8.bin' file isn't there)
+            // Build an (optional) reader for the row indices (will be null if the 'VR.u8.bin' file isn't there)
             IColumnReader rowIndexReader = TypeProviderFactory.TryGetColumnReader(streamProvider, typeof(byte), Path.Combine(columnPath, EnumWriter.RowIndexFileName), option, typeof(EnumReader));
 
             // Build a reader for the values (require caching if we we have row indices)
-            IColumnReader valueReader = TypeProviderFactory.TryGetColumnReader(streamProvider, columnType, columnPath, (rowIndexReader != null ? CachingOption.Always : option), typeof(EnumReader));
-            if (valueReader == null) throw new IOException($"No values found in {columnPath}.");
+            IColumnReader valueReader = TypeProviderFactory.GetColumnReader(streamProvider, columnType, columnPath, (rowIndexReader != null ? CachingOption.Always : option), typeof(EnumReader));
 
             // If there were row indices, wrap the column. Otherwise, return as-is.
             if (rowIndexReader != null) return new EnumReader(valueReader, rowIndexReader);

XForm/XForm/IO/TableMetadata.cs

@@ -159,5 +159,14 @@ public static bool UncachedExists(IStreamProvider streamProvider, string tableRo
         {
             return streamProvider.UncachedExists(Path.Combine(tableRootPath, MetadataFileName));
         }
+
+        public static void Delete(IStreamProvider streamProvider, string tableRootPath)
+        {
+            // Delete metadata first (the main marker we use to detect table presence)
+            streamProvider.Delete(Path.Combine(tableRootPath, MetadataFileName));
+
+            // Delete the table 
+            streamProvider.DeleteWithRetries(tableRootPath);
+        }
     }
 }

XForm/XForm/Types/ByteProvider.cs

@@ -5,6 +5,7 @@
 using System.IO;
 
 using XForm.Data;
+using XForm.Extensions;
 using XForm.IO;
 using XForm.IO.StreamProvider;
 using XForm.Types.Comparers;
@@ -22,7 +23,7 @@ public IColumnReader BinaryReader(IStreamProvider streamProvider, string columnP
             return ColumnCache.Instance.GetOrBuild(columnPath, option, () =>
             {
                 string filePath = ValuesFilePath(columnPath);
-                if (!streamProvider.Attributes(filePath).Exists) return null;
+                if (!streamProvider.UncachedExists(filePath)) return null;
                 return new ByteReader(streamProvider.OpenRead(filePath));
             });
         }

XForm/XForm/Types/PrimitiveTypeProvider.cs

@@ -6,6 +6,7 @@
 using System.Runtime.InteropServices;
 
 using XForm.Data;
+using XForm.Extensions;
 using XForm.IO;
 using XForm.IO.StreamProvider;
 using XForm.Types.Comparers;
@@ -27,7 +28,7 @@ public IColumnReader BinaryReader(IStreamProvider streamProvider, string columnP
             return ColumnCache.Instance.GetOrBuild(columnPath, option, () =>
             {
                 string filePath = ValuesFilePath(columnPath);
-                if (!streamProvider.Attributes(filePath).Exists) return null;
+                if (!streamProvider.UncachedExists(filePath)) return null;
                 return new PrimitiveArrayReader<T>(streamProvider.OpenRead(filePath));
             });
         }

XForm/XForm/Types/String8TypeProvider.cs

@@ -169,11 +169,8 @@ public String8ColumnReader(IStreamProvider streamProvider, string columnPath, Ca
             _columnPath = columnPath;
 
             _streamProvider = streamProvider;
-            _bytesReader = TypeProviderFactory.TryGetColumnReader(streamProvider, typeof(byte), Path.Combine(columnPath, "V.s.bin"), option, typeof(String8ColumnReader));
-            _positionsReader = TypeProviderFactory.TryGetColumnReader(streamProvider, typeof(int), Path.Combine(columnPath, "Vp.i32.bin"), option, typeof(String8ColumnReader));
-
-            if (_bytesReader == null) throw new IOException($"No value bytes found in {columnPath}.");
-            if (_positionsReader == null) throw new IOException($"No value positions found in {columnPath}.");
+            _bytesReader = TypeProviderFactory.GetColumnReader(streamProvider, typeof(byte), Path.Combine(columnPath, "V.s.bin"), option, typeof(String8ColumnReader));
+            _positionsReader = TypeProviderFactory.GetColumnReader(streamProvider, typeof(int), Path.Combine(columnPath, "Vp.i32.bin"), option, typeof(String8ColumnReader));
         }
 
         public int Count => _positionsReader.Count;

XForm/XForm/Types/TypeProviderFactory.cs

@@ -69,6 +69,13 @@ public static IColumnReader TryGetColumnReader(IStreamProvider streamProvider, T
             }
         }
 
+        public static IColumnReader GetColumnReader(IStreamProvider streamProvider, Type columnType, string columnPath, CachingOption option = CachingOption.AsConfigured, Type callingType = null)
+        {
+            IColumnReader reader = TryGetColumnReader(streamProvider, columnType, columnPath, option, callingType);
+            if (reader == null) throw new ColumnDataNotFoundException($"Column data not found at '{columnPath}'.", columnPath);
+            return reader;
+        }
+
         public static IColumnWriter TryGetColumnWriter(IStreamProvider streamProvider, Type columnType, string columnPath)
         {
             IColumnWriter writer = null;

XForm/XForm/XForm.csproj

@@ -102,6 +102,7 @@
     <Compile Include="Functions\Date\DateSubtract.cs" />
     <Compile Include="Functions\SimpleTransformFunction.cs" />
     <Compile Include="Functions\String\ToLower.cs" />
+    <Compile Include="IO\ColumnDataNotFoundException.cs" />
     <Compile Include="IO\VariableIntegerReaderWriter.cs" />
     <Compile Include="Verbs\Count.cs" />
     <Compile Include="Columns\DeferredArrayColumn.cs" />