diff --git a/src/dotnet/AIModel/Models/AIModelReference.cs b/src/dotnet/AIModel/Models/AIModelReference.cs index c0577de1bc..6e5f70159a 100644 --- a/src/dotnet/AIModel/Models/AIModelReference.cs +++ b/src/dotnet/AIModel/Models/AIModelReference.cs @@ -15,7 +15,7 @@ public class AIModelReference : ResourceReference /// The object type of the data source. /// [JsonIgnore] - public override Type ResourceType => + public Type AIModelType => Type switch { AIModelTypes.Basic => typeof(AIModelBase), diff --git a/src/dotnet/AIModel/ResourceProviders/AIModelResourceProviderService.cs b/src/dotnet/AIModel/ResourceProviders/AIModelResourceProviderService.cs index c5426b3f1d..54adaf8ff9 100644 --- a/src/dotnet/AIModel/ResourceProviders/AIModelResourceProviderService.cs +++ b/src/dotnet/AIModel/ResourceProviders/AIModelResourceProviderService.cs @@ -121,7 +121,7 @@ private async Task UpdateAIModel(ResourcePath reso aiModel.ObjectId = resourcePath.GetObjectId(_instanceSettings.Id, _name); - var validator = _resourceValidatorFactory.GetValidator(aiModelReference.ResourceType); + var validator = _resourceValidatorFactory.GetValidator(aiModelReference.AIModelType); if (validator is IValidator aiModelValidator) { var context = new ValidationContext(aiModel); diff --git a/src/dotnet/Agent/Models/Resources/AgentReference.cs b/src/dotnet/Agent/Models/Resources/AgentReference.cs index 8e1443d877..7e0fabf421 100644 --- a/src/dotnet/Agent/Models/Resources/AgentReference.cs +++ b/src/dotnet/Agent/Models/Resources/AgentReference.cs @@ -14,7 +14,7 @@ public class AgentReference : ResourceReference /// The object type of the agent. /// [JsonIgnore] - public override Type ResourceType => + public Type AgentType => Type switch { AgentTypes.Basic => typeof(AgentBase), diff --git a/src/dotnet/Agent/Models/Resources/AgentReferenceStore.cs b/src/dotnet/Agent/Models/Resources/AgentReferenceStore.cs new file mode 100644 index 0000000000..e7db0c70a9 --- /dev/null +++ b/src/dotnet/Agent/Models/Resources/AgentReferenceStore.cs @@ -0,0 +1,31 @@ +namespace FoundationaLLM.Agent.Models.Resources +{ + /// + /// Models the content of the agent reference store managed by the FoundationaLLM.Agent resource provider. + /// + public class AgentReferenceStore + { + /// + /// The list of all agents registered in the system. + /// + public required List AgentReferences { get; set; } + + /// + /// Creates a string-based dictionary of values from the current object. + /// + /// The string-based dictionary of values from the current object. + public Dictionary ToDictionary() => + AgentReferences.ToDictionary(ar => ar.Name); + + /// + /// Creates a new instance of the from a dictionary. + /// + /// A string-based dictionary of values. + /// The object created from the dictionary. + public static AgentReferenceStore FromDictionary(Dictionary dictionary) => + new() + { + AgentReferences = [.. dictionary.Values] + }; + } +} diff --git a/src/dotnet/Agent/ResourceProviders/AgentResourceProviderService.cs b/src/dotnet/Agent/ResourceProviders/AgentResourceProviderService.cs index 2954f5167c..dfbafb592a 100644 --- a/src/dotnet/Agent/ResourceProviders/AgentResourceProviderService.cs +++ b/src/dotnet/Agent/ResourceProviders/AgentResourceProviderService.cs @@ -242,7 +242,7 @@ private async Task UpdateAgent(ResourcePath resour StatusCodes.Status500InternalServerError); } - var validator = _resourceValidatorFactory.GetValidator(agentReference.ResourceType); + var validator = _resourceValidatorFactory.GetValidator(agentReference.AgentType); if (validator is IValidator agentValidator) { var context = new ValidationContext(agent); diff --git a/src/dotnet/DataSource/Models/DataSourceReference.cs b/src/dotnet/DataSource/Models/DataSourceReference.cs index f8e92d7ca5..1a9405e983 100644 --- a/src/dotnet/DataSource/Models/DataSourceReference.cs +++ b/src/dotnet/DataSource/Models/DataSourceReference.cs @@ -15,7 +15,7 @@ public class DataSourceReference : ResourceReference /// The object type of the data source. /// [JsonIgnore] - public override Type ResourceType => + public Type DataSourceType => Type switch { DataSourceTypes.Basic => typeof(DataSourceBase), diff --git a/src/dotnet/DataSource/Models/DataSourceReferenceStore.cs b/src/dotnet/DataSource/Models/DataSourceReferenceStore.cs new file mode 100644 index 0000000000..9c12e0fde8 --- /dev/null +++ b/src/dotnet/DataSource/Models/DataSourceReferenceStore.cs @@ -0,0 +1,35 @@ +namespace FoundationaLLM.DataSource.Models +{ + /// + /// Models the content of the data source reference store managed by the FoundationaLLM.DataSource resource provider. + /// + public class DataSourceReferenceStore + { + /// + /// The list of all data sources registered in the system. + /// + public required List DataSourceReferences { get; set; } + /// + /// The name of the default data source. + /// + public string? DefaultDataSourceName { get; set; } + + /// + /// Creates a string-based dictionary of values from the current object. + /// + /// The string-based dictionary of values from the current object. + public Dictionary ToDictionary() => + DataSourceReferences.ToDictionary(ar => ar.Name); + + /// + /// Creates a new instance of the from a dictionary. + /// + /// A string-based dictionary of values. + /// The object created from the dictionary. + public static DataSourceReferenceStore FromDictionary(Dictionary dictionary) => + new() + { + DataSourceReferences = [.. dictionary.Values] + }; + } +} diff --git a/src/dotnet/DataSource/ResourceProviders/DataSourceResourceProviderService.cs b/src/dotnet/DataSource/ResourceProviders/DataSourceResourceProviderService.cs index 807c6fe06b..19f53ff7a0 100644 --- a/src/dotnet/DataSource/ResourceProviders/DataSourceResourceProviderService.cs +++ b/src/dotnet/DataSource/ResourceProviders/DataSourceResourceProviderService.cs @@ -1,18 +1,15 @@ using Azure.Messaging; using FluentValidation; using FoundationaLLM.Common.Constants; -using FoundationaLLM.Common.Constants.Authorization; using FoundationaLLM.Common.Constants.Configuration; using FoundationaLLM.Common.Constants.ResourceProviders; using FoundationaLLM.Common.Exceptions; -using FoundationaLLM.Common.Extensions; using FoundationaLLM.Common.Interfaces; using FoundationaLLM.Common.Models.Authentication; using FoundationaLLM.Common.Models.Authorization; using FoundationaLLM.Common.Models.Configuration.Instance; using FoundationaLLM.Common.Models.Events; using FoundationaLLM.Common.Models.ResourceProviders; -using FoundationaLLM.Common.Models.ResourceProviders.Agent; using FoundationaLLM.Common.Models.ResourceProviders.DataSource; using FoundationaLLM.Common.Services.ResourceProviders; using FoundationaLLM.DataSource.Models; @@ -20,10 +17,7 @@ using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; using Microsoft.Extensions.Options; -using System.Collections.Concurrent; -using System.Text; using System.Text.Json; -using static Microsoft.IO.RecyclableMemoryStreamManager; namespace FoundationaLLM.DataSource.ResourceProviders { @@ -55,48 +49,19 @@ public class DataSourceResourceProviderService( loggerFactory.CreateLogger(), [ EventSetEventNamespaces.FoundationaLLM_ResourceProvider_DataSource - ]) + ], + useInternalReferencesStore: true) { /// protected override Dictionary GetResourceTypes() => DataSourceResourceProviderMetadata.AllowedResourceTypes; - private ConcurrentDictionary _dataSourceReferences = []; - private string _defaultDataSourceName = string.Empty; - - private const string DATA_SOURCE_REFERENCES_FILE_NAME = "_data-source-references.json"; - private const string DATA_SOURCE_REFERENCES_FILE_PATH = $"/{ResourceProviderNames.FoundationaLLM_DataSource}/{DATA_SOURCE_REFERENCES_FILE_NAME}"; - /// protected override string _name => ResourceProviderNames.FoundationaLLM_DataSource; /// - protected override async Task InitializeInternal() - { - _logger.LogInformation("Starting to initialize the {ResourceProvider} resource provider...", _name); - - if (await _storageService.FileExistsAsync(_storageContainerName, DATA_SOURCE_REFERENCES_FILE_PATH, default)) - { - var fileContent = await _storageService.ReadFileAsync(_storageContainerName, DATA_SOURCE_REFERENCES_FILE_PATH, default); - var dataSourceReferenceStore = JsonSerializer.Deserialize( - Encoding.UTF8.GetString(fileContent.ToArray())); - - _dataSourceReferences = new ConcurrentDictionary( - dataSourceReferenceStore!.ToDictionary()); - _defaultDataSourceName = dataSourceReferenceStore.DefaultDataSourceName ?? string.Empty; - } - else - { - await _storageService.WriteFileAsync( - _storageContainerName, - DATA_SOURCE_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(new DataSourceReferenceStore { DataSourceReferences = [] }), - default, - default); - } - - _logger.LogInformation("The {ResourceProvider} resource provider was successfully initialized.", _name); - } + protected override async Task InitializeInternal() => + await Task.CompletedTask; #region Resource provider support for Management API @@ -106,365 +71,97 @@ protected override async Task GetResourcesAsync( ResourcePathAuthorizationResult authorizationResult, UnifiedUserIdentity userIdentity, ResourceProviderLoadOptions? options = null) => - resourcePath.ResourceTypeInstances[0].ResourceTypeName switch - { - DataSourceResourceTypeNames.DataSources => await LoadDataSources(resourcePath.ResourceTypeInstances[0], userIdentity), - _ => throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeInstances[0].ResourceTypeName} is not supported by the {_name} resource provider.", - StatusCodes.Status400BadRequest) - }; - - #region Helpers for GetResourcesAsyncInternal - - private async Task>> LoadDataSources(ResourceTypeInstance instance, UnifiedUserIdentity userIdentity) - { - var dataSources = new List(); - - if (instance.ResourceId == null) - { - dataSources = (await Task.WhenAll(_dataSourceReferences.Values - .Where(dsr => !dsr.Deleted) - .Select(dsr => LoadDataSource(dsr)))) - .Where(ds => ds != null) - .Select(ds => ds!) - .ToList(); - } - else - { - DataSourceBase? dataSource; - if (!_dataSourceReferences.TryGetValue(instance.ResourceId, out var dataSourceReference)) - { - dataSource = await LoadDataSource(null, instance.ResourceId); - if (dataSource != null) - dataSources.Add(dataSource); - } - else - { - if (dataSourceReference.Deleted) - throw new ResourceProviderException( - $"Could not locate the {instance.ResourceId} data source resource.", - StatusCodes.Status404NotFound); - - dataSource = await LoadDataSource(dataSourceReference); - if (dataSource != null) - dataSources.Add(dataSource); - } - } - - return dataSources.Select(dataSource => new ResourceProviderGetResult - { - Resource = dataSource, - Roles = new List() - }).ToList(); - } - - private async Task LoadDataSource(DataSourceReference? dataSourceReference, string? resourceId = null) - { - if (dataSourceReference != null || !string.IsNullOrWhiteSpace(resourceId)) + resourcePath.MainResourceTypeName switch { - dataSourceReference ??= new DataSourceReference - { - Name = resourceId!, - Type = DataSourceTypes.Basic, - Filename = $"/{_name}/{resourceId}.json", - Deleted = false - }; - if (await _storageService.FileExistsAsync(_storageContainerName, dataSourceReference.Filename, default)) - { - var fileContent = await _storageService.ReadFileAsync(_storageContainerName, dataSourceReference.Filename, default); - var dataSource = JsonSerializer.Deserialize( - Encoding.UTF8.GetString(fileContent.ToArray()), - dataSourceReference.ResourceType, - _serializerSettings) as DataSourceBase - ?? throw new ResourceProviderException($"Failed to load the data source {dataSourceReference.Name}.", - StatusCodes.Status400BadRequest); - - if (!string.IsNullOrWhiteSpace(resourceId)) + DataSourceResourceTypeNames.DataSources => await LoadResources( + resourcePath.ResourceTypeInstances[0], + authorizationResult, + options ?? new ResourceProviderLoadOptions { - dataSourceReference.Type = dataSource.Type!; - _dataSourceReferences.AddOrUpdate(dataSourceReference.Name, dataSourceReference, (k, v) => dataSourceReference); - } - - return dataSource; - } - - if (string.IsNullOrWhiteSpace(resourceId)) - { - // Remove the reference from the dictionary since the file does not exist. - _dataSourceReferences.TryRemove(dataSourceReference.Name, out _); - return null; - } - } - - throw new ResourceProviderException($"The {_name} resource provider could not locate a resource because of invalid resource identification parameters.", - StatusCodes.Status400BadRequest); - } - - #endregion + IncludeRoles = resourcePath.IsResourceTypePath + }), + _ => throw new ResourceProviderException( + $"The resource type {resourcePath.MainResourceTypeName} is not supported by the {_name} resource provider.", + StatusCodes.Status400BadRequest) + }; /// protected override async Task UpsertResourceAsync(ResourcePath resourcePath, string serializedResource, UnifiedUserIdentity userIdentity) => - resourcePath.ResourceTypeInstances[0].ResourceTypeName switch + resourcePath.MainResourceTypeName switch { DataSourceResourceTypeNames.DataSources => await UpdateDataSource(resourcePath, serializedResource, userIdentity), - _ => throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeInstances[0].ResourceTypeName} is not supported by the {_name} resource provider.", - StatusCodes.Status400BadRequest) - }; - - #region Helpers for UpsertResourceAsync - - private async Task UpdateDataSource(ResourcePath resourcePath, string serializedDataSource, UnifiedUserIdentity userIdentity) - { - var dataSource = JsonSerializer.Deserialize(serializedDataSource) - ?? throw new ResourceProviderException("The object definition is invalid.", - StatusCodes.Status400BadRequest); - - if (_dataSourceReferences.TryGetValue(dataSource.Name!, out var existingDataSourceReference) - && existingDataSourceReference!.Deleted) - throw new ResourceProviderException($"The data source resource {existingDataSourceReference.Name} cannot be added or updated.", - StatusCodes.Status400BadRequest); - - if (resourcePath.ResourceTypeInstances[0].ResourceId != dataSource.Name) - throw new ResourceProviderException("The resource path does not match the object definition (name mismatch).", - StatusCodes.Status400BadRequest); - - var dataSourceReference = new DataSourceReference - { - Name = dataSource.Name!, - Type = dataSource.Type!, - Filename = $"/{_name}/{dataSource.Name}.json", - Deleted = false - }; - - dataSource.ObjectId = resourcePath.GetObjectId(_instanceSettings.Id, _name); - - var validator = _resourceValidatorFactory.GetValidator(dataSourceReference.ResourceType); - if (validator is IValidator dataSourceValidator) - { - var context = new ValidationContext(dataSource); - var validationResult = await dataSourceValidator.ValidateAsync(context); - if (!validationResult.IsValid) - { - throw new ResourceProviderException($"Validation failed: {string.Join(", ", validationResult.Errors.Select(e => e.ErrorMessage))}", - StatusCodes.Status400BadRequest); - } - } - - if (existingDataSourceReference == null) - dataSource.CreatedBy = userIdentity.UPN; - else - dataSource.UpdatedBy = userIdentity.UPN; - - await _storageService.WriteFileAsync( - _storageContainerName, - dataSourceReference.Filename, - JsonSerializer.Serialize(dataSource, _serializerSettings), - default, - default); - - _dataSourceReferences.AddOrUpdate(dataSourceReference.Name, dataSourceReference, (k, v) => dataSourceReference); - - await _storageService.WriteFileAsync( - _storageContainerName, - DATA_SOURCE_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(DataSourceReferenceStore.FromDictionary(_dataSourceReferences.ToDictionary())), - default, - default); - - return new ResourceProviderUpsertResult - { - ObjectId = (dataSource as DataSourceBase)!.ObjectId + _ => throw new ResourceProviderException( + $"The resource type {resourcePath.MainResourceTypeName} is not supported by the {_name} resource provider.", + StatusCodes.Status400BadRequest) }; - } - - #endregion /// -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - protected override async Task ExecuteActionAsync(ResourcePath resourcePath, string serializedAction, UnifiedUserIdentity userIdentity) => - resourcePath.ResourceTypeInstances.Last().ResourceTypeName switch + protected override async Task ExecuteActionAsync( + ResourcePath resourcePath, + ResourcePathAuthorizationResult authorizationResult, + string serializedAction, + UnifiedUserIdentity userIdentity) => + resourcePath.ResourceTypeName switch { - DataSourceResourceTypeNames.DataSources => resourcePath.ResourceTypeInstances.Last().Action switch + DataSourceResourceTypeNames.DataSources => resourcePath.Action switch { - ResourceProviderActions.CheckName => CheckDataSourceName(serializedAction), - ResourceProviderActions.Filter => await Filter(serializedAction), - ResourceProviderActions.Purge => await PurgeResource(resourcePath), - _ => throw new ResourceProviderException($"The action {resourcePath.ResourceTypeInstances.Last().Action} is not supported by the {_name} resource provider.", + ResourceProviderActions.CheckName => await CheckResourceName( + JsonSerializer.Deserialize(serializedAction)!), + ResourceProviderActions.Filter => await FilterResources( + resourcePath, + JsonSerializer.Deserialize(serializedAction)!, + authorizationResult, + new ResourceProviderLoadOptions + { + LoadContent = false, + IncludeRoles = false + }), + ResourceProviderActions.Purge => await PurgeResource(resourcePath), + _ => throw new ResourceProviderException( + $"The action {resourcePath.Action} is not supported by the {_name} resource provider.", StatusCodes.Status400BadRequest) }, _ => throw new ResourceProviderException() }; -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - - #region Helpers for ExecuteActionAsync - - private ResourceNameCheckResult CheckDataSourceName(string serializedAction) - { - var resourceName = JsonSerializer.Deserialize(serializedAction); - return _dataSourceReferences.Values.Any(ar => ar.Name.Equals(resourceName!.Name, StringComparison.OrdinalIgnoreCase)) - ? new ResourceNameCheckResult - { - Name = resourceName!.Name, - Type = resourceName.Type, - Status = NameCheckResultType.Denied, - Message = "A resource with the specified name already exists or was previously deleted and not purged." - } - : new ResourceNameCheckResult - { - Name = resourceName!.Name, - Type = resourceName.Type, - Status = NameCheckResultType.Allowed - }; - } - - private async Task> Filter(string serializedAction) - { - var resourceFilter = JsonSerializer.Deserialize(serializedAction) ?? - throw new ResourceProviderException("The object definition is invalid. Please provide a resource filter.", - StatusCodes.Status400BadRequest); - if (resourceFilter.Default.HasValue) - { - if (resourceFilter.Default.Value) - { - if (string.IsNullOrWhiteSpace(_defaultDataSourceName)) - throw new ResourceProviderException("The default data source is not set.", - StatusCodes.Status404NotFound); - - if (!_dataSourceReferences.TryGetValue(_defaultDataSourceName, out var dataSourceReference) - || dataSourceReference.Deleted) - throw new ResourceProviderException( - $"Could not locate the {_defaultDataSourceName} data source resource.", - StatusCodes.Status404NotFound); - - return [await LoadDataSource(dataSourceReference)]; - } - else - { - return - [ - .. (await Task.WhenAll( - _dataSourceReferences.Values - .Where(dsr => !dsr.Deleted && ( - string.IsNullOrWhiteSpace(_defaultDataSourceName) || - !dsr.Name.Equals(_defaultDataSourceName, StringComparison.OrdinalIgnoreCase))) - .Select(dsr => LoadDataSource(dsr)))) - ]; - } - } - else - { - // TODO: Apply other filters. - return - [ - .. (await Task.WhenAll( - _dataSourceReferences.Values - .Where(dsr => !dsr.Deleted) - .Select(dsr => LoadDataSource(dsr)))) - ]; - } - } - - private async Task PurgeResource(ResourcePath resourcePath) - { - var resourceName = resourcePath.ResourceTypeInstances.Last().ResourceId!; - if (_dataSourceReferences.TryGetValue(resourceName, out var agentReference)) - { - if (agentReference.Deleted) - { - // Delete the resource file from storage. - await _storageService.DeleteFileAsync( - _storageContainerName, - agentReference.Filename, - default); - - // Remove this resource reference from the store. - _dataSourceReferences.TryRemove(resourceName, out _); - - await _storageService.WriteFileAsync( - _storageContainerName, - DATA_SOURCE_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(DataSourceReferenceStore.FromDictionary(_dataSourceReferences.ToDictionary())), - default, - default); - - return new ResourceProviderActionResult(true); - } - else - { - throw new ResourceProviderException( - $"The {resourceName} data source resource is not soft-deleted and cannot be purged.", - StatusCodes.Status400BadRequest); - } - } - else - { - throw new ResourceProviderException($"Could not locate the {resourceName} data source resource.", - StatusCodes.Status404NotFound); - } - } - - #endregion /// protected override async Task DeleteResourceAsync(ResourcePath resourcePath, UnifiedUserIdentity userIdentity) { - switch (resourcePath.ResourceTypeInstances.Last().ResourceTypeName) + switch (resourcePath.ResourceTypeName) { case DataSourceResourceTypeNames.DataSources: - await DeleteDataSource(resourcePath.ResourceTypeInstances); + await DeleteResource(resourcePath); break; default: - throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeInstances.Last().ResourceTypeName} is not supported by the {_name} resource provider.", + throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeName} is not supported by the {_name} resource provider.", StatusCodes.Status400BadRequest); }; } - #region Helpers for DeleteResourceAsync + #endregion + + #region Resource provider strongly typed operations - private async Task DeleteDataSource(List instances) + /// + protected override async Task GetResourceAsyncInternal(ResourcePath resourcePath, UnifiedUserIdentity userIdentity, ResourceProviderLoadOptions? options = null) { - if (_dataSourceReferences.TryGetValue(instances.Last().ResourceId!, out var dataSourceReference)) - { - if (!dataSourceReference.Deleted) - { - dataSourceReference.Deleted = true; - - await _storageService.WriteFileAsync( - _storageContainerName, - DATA_SOURCE_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(DataSourceReferenceStore.FromDictionary(_dataSourceReferences.ToDictionary())), - default, - default); - } - } - else + switch (typeof(T)) { - throw new ResourceProviderException($"Could not locate the {instances.Last().ResourceId} data source resource.", - StatusCodes.Status404NotFound); + case Type t when t == typeof(DataSourceBase): + var apiEndpoint = await LoadResource(resourcePath.ResourceId!); + return apiEndpoint + ?? throw new ResourceProviderException( + $"The resource {resourcePath.ResourceTypeInstances[0].ResourceId!} of type {resourcePath.ResourceTypeInstances[0].ResourceType} could not be loaded.", + StatusCodes.Status500InternalServerError); + default: + throw new ResourceProviderException( + $"The resource type {typeof(T).Name} is not supported by the {_name} resource provider.", + StatusCodes.Status400BadRequest); } } #endregion - #endregion - - /// - protected override async Task GetResourceAsyncInternal(ResourcePath resourcePath, UnifiedUserIdentity userIdentity, ResourceProviderLoadOptions? options = null) where T : class { - if (resourcePath.ResourceTypeInstances.Count != 1) - throw new ResourceProviderException($"Invalid resource path"); - - if (typeof(T) != typeof(DataSourceBase)) - throw new ResourceProviderException($"The type of requested resource ({typeof(T)}) does not match the resource type specified in the path ({resourcePath.ResourceTypeInstances[0].ResourceTypeName})."); - - _dataSourceReferences.TryGetValue(resourcePath.ResourceTypeInstances[0].ResourceId!, out var dataSourceReference); - if (dataSourceReference is not null && dataSourceReference.Deleted) - throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId} of type {resourcePath.ResourceTypeInstances[0].ResourceTypeName} has been soft deleted."); - - var dataSource = await LoadDataSource(dataSourceReference, resourcePath.ResourceTypeInstances[0].ResourceId); - return dataSource as T - ?? throw new ResourceProviderException($"The resource {resourcePath.ResourceTypeInstances[0].ResourceId} of type {resourcePath.ResourceTypeInstances[0].ResourceTypeName} was not found."); - } - - #region Event handling /// @@ -489,33 +186,89 @@ protected override async Task HandleEvents(EventSetEventArgs e) private async Task HandleDataSourceResourceProviderEvent(CloudEvent e) { - if (string.IsNullOrWhiteSpace(e.Subject)) - return; + await Task.CompletedTask; + return; + + // Event handling is temporarily disabled until the updated event handling mechanism is implemented. + + //if (string.IsNullOrWhiteSpace(e.Subject)) + // return; + + //var fileName = e.Subject.Split("/").Last(); + + //_logger.LogInformation("The file [{FileName}] managed by the [{ResourceProvider}] resource provider has changed and will be reloaded.", + // fileName, _name); + + //var dataSourceReference = new DataSourceReference + //{ + // Name = Path.GetFileNameWithoutExtension(fileName), + // Filename = $"/{_name}/{fileName}", + // Type = DataSourceTypes.Basic, + // Deleted = false + //}; + + //var dataSource = await LoadDataSource(dataSourceReference); + //dataSourceReference.Name = dataSource.Name; + //dataSourceReference.Type = dataSource.Type!; + + //_dataSourceReferences.AddOrUpdate( + // dataSourceReference.Name, + // dataSourceReference, + // (k, v) => v); + + //_logger.LogInformation("The data source reference for the [{DataSourceName}] agent or type [{DataSourceType}] was loaded.", + // dataSourceReference.Name, dataSourceReference.Type); + } + + #endregion + + #region Resource management + + private async Task UpdateDataSource(ResourcePath resourcePath, string serializedDataSource, UnifiedUserIdentity userIdentity) + { + var dataSource = JsonSerializer.Deserialize(serializedDataSource) + ?? throw new ResourceProviderException("The object definition is invalid.", + StatusCodes.Status400BadRequest); - var fileName = e.Subject.Split("/").Last(); + var existingDataSourceReference = await _resourceReferenceStore!.GetResourceReference(dataSource.Name); - _logger.LogInformation("The file [{FileName}] managed by the [{ResourceProvider}] resource provider has changed and will be reloaded.", - fileName, _name); + if (resourcePath.ResourceTypeInstances[0].ResourceId != dataSource.Name) + throw new ResourceProviderException("The resource path does not match the object definition (name mismatch).", + StatusCodes.Status400BadRequest); var dataSourceReference = new DataSourceReference { - Name = Path.GetFileNameWithoutExtension(fileName), - Filename = $"/{_name}/{fileName}", - Type = DataSourceTypes.Basic, + Name = dataSource.Name!, + Type = dataSource.Type!, + Filename = $"/{_name}/{dataSource.Name}.json", Deleted = false }; - var dataSource = await LoadDataSource(dataSourceReference); - dataSourceReference.Name = dataSource.Name; - dataSourceReference.Type = dataSource.Type!; + dataSource.ObjectId = resourcePath.GetObjectId(_instanceSettings.Id, _name); + + var validator = _resourceValidatorFactory.GetValidator(dataSourceReference.DataSourceType); + if (validator is IValidator dataSourceValidator) + { + var context = new ValidationContext(dataSource); + var validationResult = await dataSourceValidator.ValidateAsync(context); + if (!validationResult.IsValid) + { + throw new ResourceProviderException($"Validation failed: {string.Join(", ", validationResult.Errors.Select(e => e.ErrorMessage))}", + StatusCodes.Status400BadRequest); + } + } - _dataSourceReferences.AddOrUpdate( - dataSourceReference.Name, - dataSourceReference, - (k, v) => v); + UpdateBaseProperties(dataSource, userIdentity, isNew: existingDataSourceReference == null); + if (existingDataSourceReference == null) + await CreateResource(dataSourceReference, dataSource); + else + await SaveResource(existingDataSourceReference, dataSource); - _logger.LogInformation("The data source reference for the [{DataSourceName}] agent or type [{DataSourceType}] was loaded.", - dataSourceReference.Name, dataSourceReference.Type); + return new ResourceProviderUpsertResult + { + ObjectId = dataSource!.ObjectId, + ResourceExists = existingDataSourceReference != null + }; } #endregion diff --git a/src/dotnet/Prompt/Models/PromptReference.cs b/src/dotnet/Prompt/Models/Resources/PromptReference.cs similarity index 90% rename from src/dotnet/Prompt/Models/PromptReference.cs rename to src/dotnet/Prompt/Models/Resources/PromptReference.cs index 8d8996fe48..0e9f707cb2 100644 --- a/src/dotnet/Prompt/Models/PromptReference.cs +++ b/src/dotnet/Prompt/Models/Resources/PromptReference.cs @@ -4,7 +4,7 @@ using FoundationaLLM.Common.Models.ResourceProviders.Prompt; using System.Text.Json.Serialization; -namespace FoundationaLLM.Prompt.Models +namespace FoundationaLLM.Prompt.Models.Resources { /// /// Provides details about a prompt. @@ -15,7 +15,7 @@ public class PromptReference : ResourceReference /// The object type of the agent. /// [JsonIgnore] - public override Type ResourceType => + public Type PromptType => Type switch { PromptTypes.Basic => typeof(PromptBase), diff --git a/src/dotnet/Prompt/Models/Resources/PromptReferenceStore.cs b/src/dotnet/Prompt/Models/Resources/PromptReferenceStore.cs new file mode 100644 index 0000000000..1986ff4233 --- /dev/null +++ b/src/dotnet/Prompt/Models/Resources/PromptReferenceStore.cs @@ -0,0 +1,31 @@ +namespace FoundationaLLM.Prompt.Models.Resources +{ + /// + /// Models the content of the prompt reference store managed by the FoundationaLLM.Prompt resource provider. + /// + public class PromptReferenceStore + { + /// + /// The list of all prompts registered in the system. + /// + public required List PromptReferences { get; set; } + + /// + /// Creates a string-based dictionary of values from the current object. + /// + /// The string-based dictionary of values from the current object. + public Dictionary ToDictionary() => + PromptReferences.ToDictionary(ar => ar.Name); + + /// + /// Creates a new instance of the from a dictionary. + /// + /// A string-based dictionary of values. + /// The object created from the dictionary. + public static PromptReferenceStore FromDictionary(Dictionary dictionary) => + new PromptReferenceStore + { + PromptReferences = dictionary.Values.ToList() + }; + } +} diff --git a/src/dotnet/Prompt/ResourceProviders/PromptResourceProviderService.cs b/src/dotnet/Prompt/ResourceProviders/PromptResourceProviderService.cs index 1fc35239d1..f178f6ff49 100644 --- a/src/dotnet/Prompt/ResourceProviders/PromptResourceProviderService.cs +++ b/src/dotnet/Prompt/ResourceProviders/PromptResourceProviderService.cs @@ -8,7 +8,7 @@ using FoundationaLLM.Common.Models.ResourceProviders; using FoundationaLLM.Common.Models.ResourceProviders.Prompt; using FoundationaLLM.Common.Services.ResourceProviders; -using FoundationaLLM.Prompt.Models; +using FoundationaLLM.Prompt.Models.Resources; using Microsoft.AspNetCore.Http; using Microsoft.Extensions.DependencyInjection; using Microsoft.Extensions.Logging; @@ -44,46 +44,20 @@ public class PromptResourceProviderService( eventService, resourceValidatorFactory, serviceProvider, - logger) + logger, + [], + useInternalReferencesStore: true) { /// protected override Dictionary GetResourceTypes() => PromptResourceProviderMetadata.AllowedResourceTypes; - private ConcurrentDictionary _promptReferences = []; - - private const string PROMPT_REFERENCES_FILE_NAME = "_prompt-references.json"; - private const string PROMPT_REFERENCES_FILE_PATH = $"/{ResourceProviderNames.FoundationaLLM_Prompt}/{PROMPT_REFERENCES_FILE_NAME}"; - /// protected override string _name => ResourceProviderNames.FoundationaLLM_Prompt; /// - protected override async Task InitializeInternal() - { - _logger.LogInformation("Starting to initialize the {ResourceProvider} resource provider...", _name); - - if (await _storageService.FileExistsAsync(_storageContainerName, PROMPT_REFERENCES_FILE_PATH, default)) - { - var fileContent = await _storageService.ReadFileAsync(_storageContainerName, PROMPT_REFERENCES_FILE_PATH, default); - var promptReferenceStore = JsonSerializer.Deserialize( - Encoding.UTF8.GetString(fileContent.ToArray())); - - _promptReferences = new ConcurrentDictionary( - promptReferenceStore!.ToDictionary()); - } - else - { - await _storageService.WriteFileAsync( - _storageContainerName, - PROMPT_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(new PromptReferenceStore { PromptReferences = [] }), - default, - default); - } - - _logger.LogInformation("The {ResourceProvider} resource provider was successfully initialized.", _name); - } + protected override async Task InitializeInternal() => + await Task.CompletedTask; #region Resource provider support for Management API @@ -93,122 +67,75 @@ protected override async Task GetResourcesAsync( ResourcePathAuthorizationResult authorizationResult, UnifiedUserIdentity userIdentity, ResourceProviderLoadOptions? options = null) => - resourcePath.ResourceTypeInstances[0].ResourceTypeName switch + resourcePath.MainResourceTypeName switch { - PromptResourceTypeNames.Prompts => await LoadPrompts(resourcePath.ResourceTypeInstances[0]), - _ => throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeInstances[0].ResourceTypeName} is not supported by the {_name} resource provider.", + PromptResourceTypeNames.Prompts => await LoadResources( + resourcePath.ResourceTypeInstances[0], + authorizationResult, + options ?? new ResourceProviderLoadOptions + { + IncludeRoles = resourcePath.IsResourceTypePath + }), + _ => throw new ResourceProviderException( + $"The resource type {resourcePath.MainResourceTypeName} is not supported by the {_name} resource provider.", StatusCodes.Status400BadRequest) }; - #region Helpers for GetResourcesAsyncInternal - - private async Task>> LoadPrompts(ResourceTypeInstance instance) - { - if (instance.ResourceId == null) + /// + protected override async Task UpsertResourceAsync(ResourcePath resourcePath, string serializedResource, UnifiedUserIdentity userIdentity) => + resourcePath.MainResourceTypeName switch { - var prompts = (await Task.WhenAll( - _promptReferences.Values - .Where(pr => !pr.Deleted) - .Select(pr => LoadPrompt(pr)))) - .Where(pr => pr != null) - .ToList(); + PromptResourceTypeNames.Prompts => await UpdatePrompt(resourcePath, serializedResource, userIdentity), + _ => throw new ResourceProviderException( + $"The resource type {resourcePath.MainResourceTypeName} is not supported by the {_name} resource provider.", + StatusCodes.Status400BadRequest), + }; - return prompts.Select(prompt => new ResourceProviderGetResult() { Resource = prompt, Roles = [] }).ToList(); - } - else + /// + protected override async Task ExecuteActionAsync( + ResourcePath resourcePath, + ResourcePathAuthorizationResult authorizationResult, + string serializedAction, + UnifiedUserIdentity userIdentity) => + resourcePath.ResourceTypeName switch { - PromptBase? prompt; - if (!_promptReferences.TryGetValue(instance.ResourceId, out var promptReference)) - { - prompt = await LoadPrompt(null, instance.ResourceId); - if (prompt != null) - { - return [new ResourceProviderGetResult() { Resource = prompt, Roles = [] }]; - } - return []; - } - - if (promptReference.Deleted) - { - throw new ResourceProviderException( - $"Could not locate the {instance.ResourceId} prompt resource.", - StatusCodes.Status404NotFound); - } - - prompt = await LoadPrompt(promptReference); - if (prompt != null) + PromptResourceTypeNames.Prompts => resourcePath.Action switch { - return [new ResourceProviderGetResult() { Resource = prompt, Roles = [] }]; - } - return []; - } - } + ResourceProviderActions.CheckName => CheckResourceName( + JsonSerializer.Deserialize(serializedAction)!), + ResourceProviderActions.Purge => await PurgeResource(resourcePath), + _ => throw new ResourceProviderException( + $"The action {resourcePath.Action} is not supported by the {_name} resource provider.", + StatusCodes.Status400BadRequest) + }, + _ => throw new ResourceProviderException() + }; - private async Task LoadPrompt(PromptReference? promptReference, string? resourceId = null) + /// + protected override async Task DeleteResourceAsync(ResourcePath resourcePath, UnifiedUserIdentity userIdentity) { - if (promptReference != null || !string.IsNullOrEmpty(resourceId)) + switch (resourcePath.ResourceTypeName) { - promptReference ??= new PromptReference - { - Name = resourceId!, - Type = PromptTypes.Multipart, - Filename = $"/{_name}/{resourceId}.json", - Deleted = false - }; - if (await _storageService.FileExistsAsync(_storageContainerName, promptReference.Filename, default)) - { - var fileContent = - await _storageService.ReadFileAsync(_storageContainerName, promptReference.Filename, default); - var prompt = JsonSerializer.Deserialize( - Encoding.UTF8.GetString(fileContent.ToArray()), - promptReference.ResourceType, - _serializerSettings) as PromptBase - ?? throw new ResourceProviderException($"Failed to load the prompt {promptReference.Name}.", - StatusCodes.Status400BadRequest); - - if (!string.IsNullOrWhiteSpace(resourceId)) - { - promptReference.Type = prompt.Type!; - _promptReferences.AddOrUpdate(promptReference.Name, promptReference, (k, v) => promptReference); - } - - return prompt; - } - - if (string.IsNullOrWhiteSpace(resourceId)) - { - // Remove the reference from the dictionary since the file does not exist. - _promptReferences.TryRemove(promptReference.Name, out _); - return null; - } - } - - throw new ResourceProviderException($"The {_name} resource provider could not locate a resource because of invalid resource identification parameters.", - StatusCodes.Status400BadRequest); + case PromptResourceTypeNames.Prompts: + await DeleteResource(resourcePath); + break; + default: + throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeName} is not supported by the {_name} resource provider.", + StatusCodes.Status400BadRequest); + }; } #endregion - /// - protected override async Task UpsertResourceAsync(ResourcePath resourcePath, string serializedResource, UnifiedUserIdentity userIdentity) => - resourcePath.ResourceTypeInstances[0].ResourceTypeName switch - { - PromptResourceTypeNames.Prompts => await UpdatePrompt(resourcePath, serializedResource, userIdentity), - _ => throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeInstances[0].ResourceTypeName} is not supported by the {_name} resource provider.", - StatusCodes.Status400BadRequest), - }; - - #region Helpers for UpsertResourceAsync + #region Resource management private async Task UpdatePrompt(ResourcePath resourcePath, string serializedPrompt, UnifiedUserIdentity userIdentity) { var prompt = JsonSerializer.Deserialize(serializedPrompt) - ?? throw new ResourceProviderException("The object definition is invalid."); + ?? throw new ResourceProviderException("The object definition is invalid.", + StatusCodes.Status400BadRequest); - if (_promptReferences.TryGetValue(prompt.Name!, out var existingPromptReference) - && existingPromptReference!.Deleted) - throw new ResourceProviderException($"The prompt resource {existingPromptReference.Name} cannot be added or updated.", - StatusCodes.Status400BadRequest); + var existingPromptReference = await _resourceReferenceStore!.GetResourceReference(prompt.Name); if (resourcePath.ResourceTypeInstances[0].ResourceId != prompt.Name) throw new ResourceProviderException("The resource path does not match the object definition (name mismatch).", @@ -222,152 +149,23 @@ private async Task UpdatePrompt(ResourcePath resou Deleted = false }; + // TODO: Add validation for the prompt object. + prompt.ObjectId = resourcePath.GetObjectId(_instanceSettings.Id, _name); + UpdateBaseProperties(prompt, userIdentity, isNew: existingPromptReference == null); if (existingPromptReference == null) - prompt.CreatedBy = userIdentity.UPN; + await CreateResource(promptReference, prompt); else - prompt.UpdatedBy = userIdentity.UPN; - - await _storageService.WriteFileAsync( - _storageContainerName, - promptReference.Filename, - JsonSerializer.Serialize(prompt, _serializerSettings), - default, - default); - - _promptReferences.AddOrUpdate(promptReference.Name, promptReference, (k, v) => v); - - await _storageService.WriteFileAsync( - _storageContainerName, - PROMPT_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(PromptReferenceStore.FromDictionary(_promptReferences.ToDictionary())), - default, - default); + await SaveResource(existingPromptReference, prompt); return new ResourceProviderUpsertResult { - ObjectId = (prompt as PromptBase)!.ObjectId - }; - } - - #endregion - - /// -#pragma warning disable CS1998 // Async method lacks 'await' operators and will run synchronously - protected override async Task ExecuteActionAsync(ResourcePath resourcePath, string serializedAction, UnifiedUserIdentity userIdentity) => - resourcePath.ResourceTypeInstances.Last().ResourceTypeName switch - { - PromptResourceTypeNames.Prompts => resourcePath.ResourceTypeInstances.Last().Action switch - { - ResourceProviderActions.CheckName => CheckPromptName(serializedAction), - ResourceProviderActions.Purge => await PurgeResource(resourcePath), - _ => throw new ResourceProviderException($"The action {resourcePath.ResourceTypeInstances.Last().Action} is not supported by the {_name} resource provider.", - StatusCodes.Status400BadRequest) - }, - _ => throw new ResourceProviderException() - }; -#pragma warning restore CS1998 // Async method lacks 'await' operators and will run synchronously - - #region Helpers for ExecuteActionAsync - - private ResourceNameCheckResult CheckPromptName(string serializedAction) - { - var resourceName = JsonSerializer.Deserialize(serializedAction); - return _promptReferences.Values.Any(ar => ar.Name == resourceName!.Name) - ? new ResourceNameCheckResult - { - Name = resourceName!.Name, - Type = resourceName.Type, - Status = NameCheckResultType.Denied, - Message = "A resource with the specified name already exists or was previously deleted and not purged." - } - : new ResourceNameCheckResult - { - Name = resourceName!.Name, - Type = resourceName.Type, - Status = NameCheckResultType.Allowed - }; - } - - private async Task PurgeResource(ResourcePath resourcePath) - { - var resourceName = resourcePath.ResourceTypeInstances.Last().ResourceId!; - if (_promptReferences.TryGetValue(resourceName, out var agentReference)) - { - if (agentReference.Deleted) - { - // Delete the resource file from storage. - await _storageService.DeleteFileAsync( - _storageContainerName, - agentReference.Filename, - default); - - // Remove this resource reference from the store. - _promptReferences.TryRemove(resourceName, out _); - - await _storageService.WriteFileAsync( - _storageContainerName, - PROMPT_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(PromptReferenceStore.FromDictionary(_promptReferences.ToDictionary())), - default, - default); - - return new ResourceProviderActionResult(true); - } - else - { - throw new ResourceProviderException( - $"The {resourceName} prompt resource is not soft-deleted and cannot be purged.", - StatusCodes.Status400BadRequest); - } - } - else - { - throw new ResourceProviderException($"Could not locate the {resourceName} prompt resource.", - StatusCodes.Status404NotFound); - } - } - - #endregion - - /// - protected override async Task DeleteResourceAsync(ResourcePath resourcePath, UnifiedUserIdentity userIdentity) - { - switch (resourcePath.ResourceTypeInstances.Last().ResourceTypeName) - { - case PromptResourceTypeNames.Prompts: - await DeletePrompt(resourcePath.ResourceTypeInstances); - break; - default: - throw new ResourceProviderException($"The resource type {resourcePath.ResourceTypeInstances.Last().ResourceTypeName} is not supported by the {_name} resource provider.", - StatusCodes.Status400BadRequest); + ObjectId = prompt!.ObjectId, + ResourceExists = existingPromptReference != null }; } - #region Helpers for DeleteResourceAsync - - private async Task DeletePrompt(List instances) - { - if (_promptReferences.TryGetValue(instances.Last().ResourceId!, out var promptReference) - && !promptReference.Deleted) - { - promptReference.Deleted = true; - - await _storageService.WriteFileAsync( - _storageContainerName, - PROMPT_REFERENCES_FILE_PATH, - JsonSerializer.Serialize(PromptReferenceStore.FromDictionary(_promptReferences.ToDictionary())), - default, - default); - } - else - throw new ResourceProviderException($"Could not locate the {instances.Last().ResourceId} agent resource.", - StatusCodes.Status404NotFound); - } - - #endregion - #endregion } }