diff --git a/src/ApiService/ApiService/ApiService.csproj b/src/ApiService/ApiService/ApiService.csproj index 9c18546e28..b166af393b 100644 --- a/src/ApiService/ApiService/ApiService.csproj +++ b/src/ApiService/ApiService/ApiService.csproj @@ -1,4 +1,4 @@ - + net6.0 v4 @@ -7,13 +7,13 @@ enable - + - + @@ -27,6 +27,7 @@ + diff --git a/src/ApiService/ApiService/Program.cs b/src/ApiService/ApiService/Program.cs index 62dc2225d5..81c37fbb9b 100644 --- a/src/ApiService/ApiService/Program.cs +++ b/src/ApiService/ApiService/Program.cs @@ -6,6 +6,7 @@ using Microsoft.Azure.Functions.Worker.Configuration; using Azure.ResourceManager.Storage.Models; using Microsoft.Extensions.DependencyInjection; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; namespace Microsoft.OneFuzz.Service; @@ -33,7 +34,8 @@ public static void Main() var host = new HostBuilder() .ConfigureFunctionsWorkerDefaults() .ConfigureServices((context, services) => - services.AddSingleton(_ => new LogTracerFactory(GetLoggers())) + services.AddSingleton(_ => new LogTracerFactory(GetLoggers())) + .AddSingleton(_ => new StorageProvider(EnvironmentVariables.OneFuzz.FuncStorage ?? throw new InvalidOperationException("Missing account id") )) ) .Build(); diff --git a/src/ApiService/ApiService/QueueNodeHearbeat.cs b/src/ApiService/ApiService/QueueNodeHearbeat.cs index d11d56379e..a1e5883d18 100644 --- a/src/ApiService/ApiService/QueueNodeHearbeat.cs +++ b/src/ApiService/ApiService/QueueNodeHearbeat.cs @@ -6,41 +6,46 @@ using Azure.Data.Tables; using System.Threading.Tasks; using Azure; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; namespace Microsoft.OneFuzz.Service; -enum HeartbeatType -{ - MachineAlive, - TaskAlive, -} - -record NodeHeartbeatEntry(string NodeId, Dictionary[] data); - - public class QueueNodeHearbeat { - private readonly ILogger _logger; + private readonly IStorageProvider _storageProvider; - public QueueNodeHearbeat(ILoggerFactory loggerFactory) + public QueueNodeHearbeat(ILoggerFactory loggerFactory, IStorageProvider storageProvider) { _logger = loggerFactory.CreateLogger(); + _storageProvider = storageProvider; } [Function("QueueNodeHearbeat")] - public void Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) + public async Task Run([QueueTrigger("myqueue-items", Connection = "AzureWebJobsStorage")] string msg) { - var hb = JsonSerializer.Deserialize(msg); - + var hb = JsonSerializer.Deserialize(msg, EntityConverter.GetJsonSerializerOptions()); + var node = await Node.GetByMachineId(_storageProvider, hb.NodeId); - _logger.LogInformation($"heartbeat: {msg}"); - } -} - + if (node == null) { + _logger.LogWarning($"invalid node id: {hb.NodeId}"); + return; + } + var newNode = node with { Heartbeat = DateTimeOffset.UtcNow }; + await _storageProvider.Replace(newNode); + //send_event( + // EventNodeHeartbeat( + // machine_id = node.machine_id, + // scaleset_id = node.scaleset_id, + // pool_name = node.pool_name, + // ) + //) + _logger.LogInformation($"heartbeat: {msg}"); + } +} diff --git a/src/ApiService/ApiService/model.cs b/src/ApiService/ApiService/model.cs index 21b784c50b..8e6f6df290 100644 --- a/src/ApiService/ApiService/model.cs +++ b/src/ApiService/ApiService/model.cs @@ -1,87 +1,87 @@ -using Azure.Data.Tables; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; using System; -using System.Runtime.Serialization; +using System.Collections.Generic; -namespace ApiService; +namespace Microsoft.OneFuzz.Service; -record NodeCommandStopIfFree { } +/// Convention for database entities: +/// All entities are represented by immutable records +/// All database entities need to derive from EntityBase +/// Only properties that also apears as parameter initializers are mapped to the database +/// The name of the property will be tranlated to snake case and used as the column name +/// It is possible to rename the column name by using the [property:JsonPropertyName("column_name")] attribute +/// the "partion key" and "row key" are identified by the [PartitionKey] and [RowKey] attributes +/// Guids are mapped to string in the db -record StopNodeCommand{} -record StopTaskNodeCommand{ - Guid TaskId; +[SkipRename] +public enum HeartbeatType +{ + MachineAlive, + TaskAlive, } -record NodeCommandAddSshKey{ - string PublicKey; -} +public record HeartbeatData(HeartbeatType type); -record NodeCommand -{ - StopNodeCommand? Stop; - StopTaskNodeCommand? StopTask; - NodeCommandAddSshKey? AddSshKey; - NodeCommandStopIfFree? StopIfFree; -} +public record NodeHeartbeatEntry(Guid NodeId, HeartbeatData[] data); -enum NodeTaskState -{ - init, - setting_up, - running, -} +public record NodeCommandStopIfFree(); -record NodeTasks -{ - Guid MachineId; - Guid TaskId; - NodeTaskState State = NodeTaskState.init; +public record StopNodeCommand(); +public record StopTaskNodeCommand(Guid TaskId); + +public record NodeCommandAddSshKey(string PublicKey); + + +public record NodeCommand +( + StopNodeCommand? Stop, + StopTaskNodeCommand? StopTask, + NodeCommandAddSshKey? AddSshKey, + NodeCommandStopIfFree? StopIfFree +); + +public enum NodeTaskState +{ + Init, + SettingUp, + Running, } -enum NodeState +public record NodeTasks +( + Guid MachineId, + Guid TaskId, + NodeTaskState State = NodeTaskState.Init +); + +public enum NodeState { - init, + Init, free, - setting_up, - rebooting, - ready, - busy, - done, - shutdown, - halt, + SettingUp, + Rebooting, + Ready, + Busy, + Done, + Shutdown, + Halt, } -record Node : ITableEntity -{ - [DataMember(Name = "initialized_at")] - public DateTimeOffset? InitializedAt; - [DataMember(Name = "pool_name")] - public string PoolName; - [DataMember(Name = "pool_id")] - public Guid? PoolId; - [DataMember(Name = "machine_id")] - public Guid MachineId; - [DataMember(Name = "state")] - public NodeState State; - [DataMember(Name = "scaleset_id")] - public Guid? ScalesetId; - [DataMember(Name = "heartbeat")] - public DateTimeOffset Heartbeat; - [DataMember(Name = "version")] - public Version Version; - [DataMember(Name = "reimage_requested")] - public bool ReimageRequested; - [DataMember(Name = "delete_requested")] - public bool DeleteRequested; - [DataMember(Name = "debug_keep_node")] - public bool DebugKeepNode; - - public string PartitionKey { get => PoolName; set => PoolName = value; } - public string RowKey { get => MachineId.ToString(); set => MachineId = Guid.Parse(value); } - public Azure.ETag ETag { get; set; } - DateTimeOffset? ITableEntity.Timestamp { get; set; } - -} \ No newline at end of file +public partial record Node +( + DateTimeOffset? InitializedAt, + [PartitionKey] string PoolName, + Guid? PoolId, + [RowKey] Guid MachineId, + NodeState State, + Guid? ScalesetId, + DateTimeOffset Heartbeat, + Version Version, + bool ReimageRequested, + bool DeleteRequested, + bool DebugKeepNode +): EntityBase(); diff --git a/src/ApiService/ApiService/onefuzzlib/Nodes.cs b/src/ApiService/ApiService/onefuzzlib/Nodes.cs new file mode 100644 index 0000000000..dd5c39ef86 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/Nodes.cs @@ -0,0 +1,17 @@ +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +using System; +using System.Linq; +using System.Threading.Tasks; + +namespace Microsoft.OneFuzz.Service; + +public partial record Node +{ + public async static Task GetByMachineId(IStorageProvider storageProvider, Guid machineId) { + var tableClient = await storageProvider.GetTableClient("Node"); + + var data = storageProvider.QueryAsync(filter: $"RowKey eq '{machineId}'"); + + return await data.FirstOrDefaultAsync(); + } +} diff --git a/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs b/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs index 71a3e19cdd..c0966870de 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/CaseConverter.cs @@ -2,53 +2,56 @@ using System.Collections.Generic; using System.Linq; -namespace ApiService.onefuzzlib.orm +namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +public class CaseConverter { - public class CaseConverter + /// get the start indices of each word and the lat indice + static IEnumerable getIndices(string input) { - /// get the start indices of each word and the lat indice - static IEnumerable getIndices(string input) + + yield return 0; + for (var i = 1; i < input.Length; i++) { - yield return 0; - for (var i = 1; i < input.Length; i++) + if (Char.IsDigit(input[i])) { + continue; + } - if (Char.IsDigit(input[i])) - { + if (i < input.Length - 1 && Char.IsDigit(input[i + 1])) + { + continue; + } + + // is the current letter uppercase + if (Char.IsUpper(input[i])) + { + if (input[i - 1] == '_') { continue; } - if (i < input.Length - 1 && Char.IsDigit(input[i + 1])) + if (i < input.Length - 1 && !Char.IsUpper(input[i + 1])) { - continue; + yield return i; } - - // is the current letter uppercase - if (Char.IsUpper(input[i])) + else if (!Char.IsUpper(input[i - 1])) { - if (i < input.Length - 1 && !Char.IsUpper(input[i + 1])) - { - yield return i; - } - else if (!Char.IsUpper(input[i - 1])) - { - yield return i; - } + yield return i; } } - yield return input.Length; } + yield return input.Length; + } - public static string PascalToSnake(string input) - { - var indices = getIndices(input).ToArray(); - return string.Join("_", Enumerable.Zip(indices, indices.Skip(1)).Select(x => input.Substring(x.First, x.Second - x.First).ToLowerInvariant())); + public static string PascalToSnake(string input) + { + var indices = getIndices(input).ToArray(); + return string.Join("_", Enumerable.Zip(indices, indices.Skip(1)).Select(x => input.Substring(x.First, x.Second - x.First).ToLowerInvariant())); - } - public static string SnakeToPascal(string input) - { - return string.Join("", input.Split('_', StringSplitOptions.RemoveEmptyEntries).Select(x => $"{Char.ToUpper(x[0])}{x.Substring(1)}")); - } + } + public static string SnakeToPascal(string input) + { + return string.Join("", input.Split('_', StringSplitOptions.RemoveEmptyEntries).Select(x => $"{Char.ToUpper(x[0])}{x.Substring(1)}")); } } diff --git a/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs b/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs index b6417d73ce..2122c737ad 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/CustomConverterFactory.cs @@ -7,161 +7,163 @@ using System.Text.Json; using System.Text.Json.Serialization; -namespace ApiService.onefuzzlib.orm +namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + +public sealed class CustomEnumConverterFactory : JsonConverterFactory { - public sealed class CustomEnumConverterFactory : JsonConverterFactory + public override bool CanConvert(Type typeToConvert) => typeToConvert.IsEnum; + + public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) { - public override bool CanConvert(Type typeToConvert) => typeToConvert.IsEnum; + object[]? knownValues = null; - public override JsonConverter CreateConverter(Type typeToConvert, JsonSerializerOptions options) + if (typeToConvert == typeof(BindingFlags)) { - object[]? knownValues = null; - - if (typeToConvert == typeof(BindingFlags)) - { - knownValues = new object[] { BindingFlags.CreateInstance | BindingFlags.DeclaredOnly }; - } - - return (JsonConverter)Activator.CreateInstance( - typeof(CustomEnumConverter<>).MakeGenericType(typeToConvert), - BindingFlags.Instance | BindingFlags.Public, - binder: null, - args: new object?[] { options.PropertyNamingPolicy, options, knownValues }, - culture: null)!; + knownValues = new object[] { BindingFlags.CreateInstance | BindingFlags.DeclaredOnly }; } + + return (JsonConverter)Activator.CreateInstance( + typeof(CustomEnumConverter<>).MakeGenericType(typeToConvert), + BindingFlags.Instance | BindingFlags.Public, + binder: null, + args: new object?[] { options.PropertyNamingPolicy, options, knownValues }, + culture: null)!; } +} - public sealed class CustomEnumConverter : JsonConverter where T : Enum - { - private readonly JsonNamingPolicy _namingPolicy; +public sealed class CustomEnumConverter : JsonConverter where T : Enum +{ + private readonly JsonNamingPolicy _namingPolicy; - private readonly Dictionary _readCache = new(); - private readonly Dictionary _writeCache = new(); + private readonly Dictionary _readCache = new(); + private readonly Dictionary _writeCache = new(); - // This converter will only support up to 64 enum values (including flags) on serialization and deserialization - private const int NameCacheLimit = 64; + // This converter will only support up to 64 enum values (including flags) on serialization and deserialization + private const int NameCacheLimit = 64; - private const string ValueSeparator = ","; + private const string ValueSeparator = ","; - public CustomEnumConverter(JsonNamingPolicy namingPolicy, JsonSerializerOptions options, object[]? knownValues) - { - _namingPolicy = namingPolicy; + public CustomEnumConverter(JsonNamingPolicy namingPolicy, JsonSerializerOptions options, object[]? knownValues) + { + _namingPolicy = namingPolicy; - bool continueProcessing = true; - for (int i = 0; i < knownValues?.Length; i++) + bool continueProcessing = true; + for (int i = 0; i < knownValues?.Length; i++) + { + if (!TryProcessValue((T)knownValues[i])) { - if (!TryProcessValue((T)knownValues[i])) - { - continueProcessing = false; - break; - } + continueProcessing = false; + break; } + } + + var type = typeof(T); + var skipFormat = type.GetCustomAttribute() != null; + if (continueProcessing) + { + Array values = Enum.GetValues(type); - if (continueProcessing) + for (int i = 0; i < values.Length; i++) { - Array values = Enum.GetValues(typeof(T)); + T value = (T)values.GetValue(i)!; - for (int i = 0; i < values.Length; i++) + if (!TryProcessValue(value, skipFormat)) { - T value = (T)values.GetValue(i)!; - - if (!TryProcessValue(value)) - { - break; - } + break; } } + } - bool TryProcessValue(T value) + bool TryProcessValue(T value, bool skipFormat=false) + { + if (_readCache.Count == NameCacheLimit) { - if (_readCache.Count == NameCacheLimit) - { - Debug.Assert(_writeCache.Count == NameCacheLimit); - return false; - } - - FormatAndAddToCaches(value, options.Encoder); - return true; + Debug.Assert(_writeCache.Count == NameCacheLimit); + return false; } + + FormatAndAddToCaches(value, options.Encoder, skipFormat); + return true; } + } + + public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + { + string? json; - public override T Read(ref Utf8JsonReader reader, Type typeToConvert, JsonSerializerOptions options) + if (reader.TokenType != JsonTokenType.String || (json = reader.GetString()) == null) { - string? json; + throw new JsonException(); + } - if (reader.TokenType != JsonTokenType.String || (json = reader.GetString()) == null) + var value = json.Split(ValueSeparator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) + .Select(x => { - throw new JsonException(); - } - - var value = json.Split(ValueSeparator, StringSplitOptions.RemoveEmptyEntries | StringSplitOptions.TrimEntries) - .Select(x => + if (!_readCache.TryGetValue(x, out T value)) { - if (!_readCache.TryGetValue(x, out T value)) - { - throw new JsonException(); - } - return value; + throw new JsonException(); + } + return value; - }).ToArray(); + }).ToArray(); - if (value.Length == 1) - { - return value[0]; - } + if (value.Length == 1) + { + return value[0]; + } - var result = default(T); + var result = default(T); - return (T)(object)value.Aggregate(0, (state, value) => (int)(object)state | (int)(object)value); - } + return (T)(object)value.Aggregate(0, (state, value) => (int)(object)state | (int)(object)value); + } - public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + public override void Write(Utf8JsonWriter writer, T value, JsonSerializerOptions options) + { + if (!_writeCache.TryGetValue(value, out JsonEncodedText formatted)) { - if (!_writeCache.TryGetValue(value, out JsonEncodedText formatted)) + if (_writeCache.Count == NameCacheLimit) { - if (_writeCache.Count == NameCacheLimit) - { - Debug.Assert(_readCache.Count == NameCacheLimit); - throw new ArgumentOutOfRangeException(); - } - - formatted = FormatAndAddToCaches(value, options.Encoder); + Debug.Assert(_readCache.Count == NameCacheLimit); + throw new ArgumentOutOfRangeException(); } - writer.WriteStringValue(formatted); + formatted = FormatAndAddToCaches(value, options.Encoder); } - private JsonEncodedText FormatAndAddToCaches(T value, JavaScriptEncoder? encoder) + writer.WriteStringValue(formatted); + } + + private JsonEncodedText FormatAndAddToCaches(T value, JavaScriptEncoder? encoder, bool skipFormat = false) + { + (string valueFormattedToStr, JsonEncodedText valueEncoded) = FormatEnumValue(value.ToString(), _namingPolicy, encoder, skipFormat); + _readCache[valueFormattedToStr] = value; + _writeCache[value] = valueEncoded; + return valueEncoded; + } + + private ValueTuple FormatEnumValue(string value, JsonNamingPolicy namingPolicy, JavaScriptEncoder? encoder, bool skipFormat = false) + { + string converted; + + if (!value.Contains(ValueSeparator)) { - (string valueFormattedToStr, JsonEncodedText valueEncoded) = FormatEnumValue(value.ToString(), _namingPolicy, encoder); - _readCache[valueFormattedToStr] = value; - _writeCache[value] = valueEncoded; - return valueEncoded; + converted = skipFormat ? value : namingPolicy.ConvertName(value); } - - private ValueTuple FormatEnumValue(string value, JsonNamingPolicy namingPolicy, JavaScriptEncoder? encoder) + else { - string converted; + // todo: optimize implementation here by leveraging https://github.com/dotnet/runtime/issues/934. + string[] enumValues = value.Split(ValueSeparator); - if (!value.Contains(ValueSeparator)) + for (int i = 0; i < enumValues.Length; i++) { - converted = namingPolicy.ConvertName(value); + var trimmed = enumValues[i].Trim(); + enumValues[i] = skipFormat? trimmed : namingPolicy.ConvertName(trimmed); } - else - { - // todo: optimize implementation here by leveraging https://github.com/dotnet/runtime/issues/934. - string[] enumValues = value.Split(ValueSeparator); - - for (int i = 0; i < enumValues.Length; i++) - { - enumValues[i] = namingPolicy.ConvertName(enumValues[i].Trim()); - } - converted = string.Join(ValueSeparator, enumValues); - } - - return (converted, JsonEncodedText.Encode(converted, encoder)); + converted = string.Join(ValueSeparator, enumValues); } + + return (converted, JsonEncodedText.Encode(converted, encoder)); } } diff --git a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs index f8115a7bcb..81b4b9f37d 100644 --- a/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs +++ b/src/ApiService/ApiService/onefuzzlib/orm/Orm.cs @@ -4,15 +4,22 @@ using System.Linq; using System.Linq.Expressions; using System.Text.Json; -using ApiService; -using System.Threading.Tasks; -using System.Collections.Generic; -using ApiService.onefuzzlib.orm; using System.Text.Json.Serialization; using System.Collections.Concurrent; +using Azure; -namespace Microsoft.OneFuzz.Service; +namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; +public abstract record EntityBase +{ + public ETag? ETag { get; set; } + public DateTimeOffset? TimeStamp { get; set; } + +} + +/// Indicates that the enum cases should no be renamed +[AttributeUsage(AttributeTargets.Enum)] +public class SkipRename : Attribute { } public class RowKeyAttribute : Attribute { } public class PartitionKeyAttribute : Attribute { } @@ -41,12 +48,18 @@ public class EntityConverter public EntityConverter() { - _options = new JsonSerializerOptions() + _options = GetJsonSerializerOptions(); + _cache = new ConcurrentDictionary(); + } + + + public static JsonSerializerOptions GetJsonSerializerOptions() { + var options = new JsonSerializerOptions() { PropertyNamingPolicy = new OnefuzzNamingPolicy(), }; - _options.Converters.Add(new CustomEnumConverterFactory()); - _cache = new ConcurrentDictionary(); + options.Converters.Add(new CustomEnumConverterFactory()); + return options; } internal Func BuildConstructerFrom(ConstructorInfo constructorInfo) @@ -109,7 +122,7 @@ private EntityInfo GetEntityInfo() }); } - public TableEntity ToTableEntity(T typedEntity) + public TableEntity ToTableEntity(T typedEntity) where T: EntityBase { if (typedEntity == null) { @@ -163,11 +176,15 @@ public TableEntity ToTableEntity(T typedEntity) } + if (typedEntity.ETag.HasValue) { + tableEntity.ETag = typedEntity.ETag.Value; + } + return tableEntity; } - public T ToRecord(TableEntity entity) + public T ToRecord(TableEntity entity) where T: EntityBase { var entityInfo = GetEntityInfo(); var parameters = @@ -235,30 +252,13 @@ public T ToRecord(TableEntity entity) } ).ToArray(); - return (T)entityInfo.constructor.Invoke(parameters); - } - + var entityRecord = (T)entityInfo.constructor.Invoke(parameters); + entityRecord.ETag = entity.ETag; + entityRecord.TimeStamp = entity.Timestamp; - public interface IStorageProvider - { - Task GetStorageClient(string table, string accounId); + return entityRecord; } - public class StorageProvider : IStorageProvider - { - public async Task GetStorageClient(string table, string accounId) - { - var (name, key) = GetStorageAccountNameAndKey(accounId); - var tableClient = new TableServiceClient(new Uri(accounId), new TableSharedKeyCredential(name, key)); - await tableClient.CreateTableIfNotExistsAsync(table); - return tableClient; - } - - private (string, string) GetStorageAccountNameAndKey(string accounId) - { - throw new NotImplementedException(); - } - } } diff --git a/src/ApiService/ApiService/onefuzzlib/orm/StorageProvider.cs b/src/ApiService/ApiService/onefuzzlib/orm/StorageProvider.cs new file mode 100644 index 0000000000..68673c47e8 --- /dev/null +++ b/src/ApiService/ApiService/onefuzzlib/orm/StorageProvider.cs @@ -0,0 +1,68 @@ +using Azure.Data.Tables; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Threading.Tasks; +using Azure.Core; +using Azure.ResourceManager.Storage; +using Azure.ResourceManager; +using Azure.Identity; + +namespace Microsoft.OneFuzz.Service.OneFuzzLib.Orm; + + +public interface IStorageProvider +{ + Task GetTableClient(string table); + IAsyncEnumerable QueryAsync(string filter) where T : EntityBase; + Task Replace(T entity) where T : EntityBase; + +} + +public class StorageProvider : IStorageProvider +{ + private readonly string _accountId; + private readonly EntityConverter _entityConverter; + private readonly ArmClient _armClient; + + public StorageProvider(string accountId) { + _accountId = accountId; + _entityConverter = new EntityConverter(); + _armClient = new ArmClient(new DefaultAzureCredential()); + } + + public async Task GetTableClient(string table) + { + var (name, key) = GetStorageAccountNameAndKey(_accountId); + var identifier = new ResourceIdentifier(_accountId); + var tableClient = new TableServiceClient(new Uri($"https://{identifier.Name}.table.core.windows.net"), new TableSharedKeyCredential(name, key)); + await tableClient.CreateTableIfNotExistsAsync(table); + return tableClient.GetTableClient(table); + } + + + public (string?, string?) GetStorageAccountNameAndKey(string accountId) { + var resourceId = new ResourceIdentifier(accountId); + var storageAccount = _armClient.GetStorageAccount(resourceId); + var key = storageAccount.GetKeys().Value.Keys.FirstOrDefault(); + return (resourceId.Name, key?.Value); + } + + public async IAsyncEnumerable QueryAsync(string filter) where T : EntityBase + { + var tableClient = await GetTableClient(typeof(T).Name); + + await foreach (var x in tableClient.QueryAsync(filter).Select(x => _entityConverter.ToRecord(x))) { + yield return x; + } + } + + public async Task Replace(T entity) where T : EntityBase + { + var tableClient = await GetTableClient(typeof(T).Name); + var tableEntity = _entityConverter.ToTableEntity(entity); + var response = await tableClient.UpsertEntityAsync(tableEntity); + return !response.IsError; + + } +} \ No newline at end of file diff --git a/src/ApiService/ApiService/packages.lock.json b/src/ApiService/ApiService/packages.lock.json index 74edf76bc5..eb9f2dfe12 100644 --- a/src/ApiService/ApiService/packages.lock.json +++ b/src/ApiService/ApiService/packages.lock.json @@ -109,13 +109,13 @@ }, "Microsoft.Azure.Functions.Worker": { "type": "Direct", - "requested": "[1.5.2, )", - "resolved": "1.5.2", - "contentHash": "a42hCpHj4hb/SddLfTcnf+Vm9pbHZ8jSrKqopCBIoq6KaWWz5aAiaVTE1gv8YJqQern7RP8Z5WSn6T4a0wJsfg==", + "requested": "[1.6.0, )", + "resolved": "1.6.0", + "contentHash": "Gzq2IPcMCym6wPpFayLbuvhrfr72OEInJJlKaIAqU9+HldVaTt54cm3hPe7kIok+QuWnwb/TtYtlmrkR0Nbhsg==", "dependencies": { "Azure.Core": "1.10.0", - "Microsoft.Azure.Functions.Worker.Core": "1.3.1", - "Microsoft.Azure.Functions.Worker.Grpc": "1.3.0", + "Microsoft.Azure.Functions.Worker.Core": "1.4.0", + "Microsoft.Azure.Functions.Worker.Grpc": "1.3.1", "Microsoft.Extensions.Hosting": "5.0.0", "Microsoft.Extensions.Hosting.Abstractions": "5.0.0" } @@ -140,11 +140,12 @@ }, "Microsoft.Azure.Functions.Worker.Extensions.Storage": { "type": "Direct", - "requested": "[4.0.4, )", - "resolved": "4.0.4", - "contentHash": "lg/sBDVPkXI0l1kHYbvyF1eA+gwaM3p/WgjX1nwVMyAkJpniDCNa6Rv/nxJwLvdWFzFLOuUp7ns0sxF1AjNuhg==", + "requested": "[5.0.0, )", + "resolved": "5.0.0", + "contentHash": "VwcmWk29//8CkXCxCR7vxMnqsuh+O0018eavRCGscI+uRKHdvlHDG97vwfdwuTzwKuoo7ztQbAvHfkp+sxoiEQ==", "dependencies": { - "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "1.0.0" + "Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs": "5.0.0", + "Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues": "5.0.0" } }, "Microsoft.Azure.Functions.Worker.Extensions.Timer": { @@ -213,6 +214,15 @@ "Microsoft.IdentityModel.Tokens": "6.16.0" } }, + "System.Linq.Async": { + "type": "Direct", + "requested": "[6.0.1, )", + "resolved": "6.0.1", + "contentHash": "0YhHcaroWpQ9UCot3Pizah7ryAzQhNvobLMSxeDIGmnXfkQn8u5owvpOH0K6EVB+z9L7u6Cc4W17Br/+jyttEQ==", + "dependencies": { + "Microsoft.Bcl.AsyncInterfaces": "6.0.0" + } + }, "Google.Protobuf": { "type": "Transitive", "resolved": "3.15.8", @@ -291,8 +301,8 @@ }, "Microsoft.Azure.Functions.Worker.Core": { "type": "Transitive", - "resolved": "1.3.1", - "contentHash": "BPx6MPCpxAOHEWqNvZaHcDiZWTyDUZXnYa6np3NqQ0oVq/mPPBYWm5EZfOHywaSkiV+yrePxIO/emochtRMbLA==", + "resolved": "1.4.0", + "contentHash": "6fTSb6JDm+1CNKsaPziL36c3tfN4xxYnC9XoJsm0g9tY+72dVqUa2aPc6RtkwBmT5sjNrsUDlUC+IhG+ehjppQ==", "dependencies": { "Azure.Core": "1.10.0", "Microsoft.Extensions.Hosting": "5.0.0", @@ -301,19 +311,35 @@ }, "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": { "type": "Transitive", - "resolved": "1.0.0", - "contentHash": "9dPxxE6KdBXM4JEiwfs6W+DuZ+7UAE0Ea2pSmSTQ/e954HDVfcEcSyIopZujjK1jbwExSBbBQHwn6VrA2Dpx2A==" + "resolved": "1.1.0", + "contentHash": "kAs9BTuzdOvyuN2m5CYyQzyvzXKJ6hhIOgcwm0W8Q+Fwj91a1eBmRSi9pVzpM4V3skNt/+pkPD3wxFD4nEw0bg==" + }, + "Microsoft.Azure.Functions.Worker.Extensions.Storage.Blobs": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "Cr+ziBQA/Lyt5rMURHBje+foGook+B82gnAfEE32bHcXGlpKJnnVdLqHy0OqHliUksFddkRYS2gY8dYx+NH/mA==", + "dependencies": { + "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "1.1.0" + } + }, + "Microsoft.Azure.Functions.Worker.Extensions.Storage.Queues": { + "type": "Transitive", + "resolved": "5.0.0", + "contentHash": "cF95kiiU6PD9sptrV3GKQKzRv2DYATYNTpOtvUtbAYQ4xPFKgF4ke3fDBcu+cu2O1/C8FQ7MhzkEQv00bx552A==", + "dependencies": { + "Microsoft.Azure.Functions.Worker.Extensions.Abstractions": "1.1.0" + } }, "Microsoft.Azure.Functions.Worker.Grpc": { "type": "Transitive", - "resolved": "1.3.0", - "contentHash": "bxveZ9OKBPyzztl7tTClAAdw6PNjLiQ2fEboafAXcIF7+2igHwp/RlUILfXLrzgqrbWPkFfCRCpSJ3LIIHYaJQ==", + "resolved": "1.3.1", + "contentHash": "lMlbyfRagSQZVWN73jnaB0tVTMhfKHSy6IvFXC7fCGh+7uA9LJNjcMOQbVkUnmvb/I/SxslMqD7xcebrxFL3TQ==", "dependencies": { "Azure.Core": "1.10.0", "Google.Protobuf": "3.15.8", "Grpc.Net.Client": "2.37.0", "Grpc.Net.ClientFactory": "2.37.0", - "Microsoft.Azure.Functions.Worker.Core": "1.2.0", + "Microsoft.Azure.Functions.Worker.Core": "1.3.1", "Microsoft.Extensions.Hosting": "5.0.0", "Microsoft.Extensions.Hosting.Abstractions": "5.0.0" } @@ -325,8 +351,8 @@ }, "Microsoft.Bcl.AsyncInterfaces": { "type": "Transitive", - "resolved": "1.1.1", - "contentHash": "yuvf07qFWFqtK3P/MRkEKLhn5r2UbSpVueRziSqj0yJQIKFwG1pq9mOayK3zE5qZCTs0CbrwL9M6R8VwqyGy2w==" + "resolved": "6.0.0", + "contentHash": "UcSjPsst+DfAdJGVDsu346FX0ci0ah+lw3WRtn18NUwEqRt70HaOQ7lI72vy3+1LxtqI3T5GWwV39rQSrCzAeg==" }, "Microsoft.CSharp": { "type": "Transitive", diff --git a/src/ApiService/Tests/OrmTest.cs b/src/ApiService/Tests/OrmTest.cs index be2b86c14a..4a24bb9624 100644 --- a/src/ApiService/Tests/OrmTest.cs +++ b/src/ApiService/Tests/OrmTest.cs @@ -3,8 +3,8 @@ using Microsoft.OneFuzz.Service; using Azure.Data.Tables; using System.Text.Json.Nodes; -using ApiService.onefuzzlib.orm; using System.Text.Json.Serialization; +using Microsoft.OneFuzz.Service.OneFuzzLib.Orm; namespace Tests { @@ -41,8 +41,7 @@ record Entity1( TestFlagEnum TheFlag, [property:JsonPropertyName("a__special__name")] string Renamed, TestObject TheObject - - ); + ): EntityBase(); [Fact] @@ -128,7 +127,10 @@ public void TestConvertPascalToSnakeCase() ("AString", "a_string"), ("Some4Numbers234", "some4_numbers234"), ("TEST123String", "test123_string"), - ("TheTwo", "the_two") + ("TheTwo", "the_two"), + ("___Value2", "___value2"), + ("V_A_L_U_E_3", "v_a_l_u_e_3"), + ("ALLCAPS", "allcaps"), }; foreach (var (input, expected) in testCases) @@ -151,7 +153,7 @@ public void TestConvertSnakeToPAscalCase() ("a_string" , "AString"), ("some4_numbers234" , "Some4Numbers234"), ("test123_string" , "Test123String"), - ("the_two" , "TheTwo"), + ("the_two" , "TheTwo") }; foreach (var (input, expected) in testCases) @@ -160,6 +162,5 @@ public void TestConvertSnakeToPAscalCase() Assert.Equal(expected, actual); } } - } } \ No newline at end of file