diff --git a/.github/workflows/documentation.yml b/.github/workflows/documentation.yml index e22c1a1b..a47250c4 100644 --- a/.github/workflows/documentation.yml +++ b/.github/workflows/documentation.yml @@ -1,4 +1,4 @@ -name: Auto - Documentation +name: Auto - Doc on: # Trigger spell check on pull requests diff --git a/bicep/modules/aks_cluster.bicep b/bicep/modules/aks_cluster.bicep index c3fb1cc2..1910d35d 100644 --- a/bicep/modules/aks_cluster.bicep +++ b/bicep/modules/aks_cluster.bicep @@ -336,7 +336,6 @@ var outboundTrafficType = aksOutboundTrafficType=='natGateway' ? ( custom_vnet ? @description('System Pool presets are derived from the recommended system pool specs') var systemPoolPresets = { - // 4 vCPU, 16 GiB RAM, 32 GiB Temp Disk, (3600) IOPS, 128 GB Managed OS Disk Burstable : { vmSize: 'Standard_B2ms' minCount: 1 @@ -350,7 +349,6 @@ var systemPoolPresets = { osDiskSize: 128 maxPods: 30 } - // 2 vCPU, 7 GiB RAM, 14 GiB SSD, (8000) IOPS, 128 GB Managed OS Disk Standard : { vmSize: 'Standard_D2s_v5' minCount: 1 @@ -393,10 +391,9 @@ var systemPoolProfile = { @description('First User Pool presets') var userPoolPresets = { - // 4 vCPU, 16 GiB RAM, 32 GiB Temp Disk, (3600) IOPS, 128 GB Managed OS Disk Burstable : { vmSize: 'Standard_B4ms' - minCount: 3 + minCount: 4 maxCount: 20 availabilityZones: [ '1' @@ -405,12 +402,11 @@ var userPoolPresets = { ] osDiskType: 'Managed' osDiskSize: 128 - maxPods: 30 + maxPods: 60 } - // 4 vCPU, 32 GiB RAM, 64 GiB SSD, (8000) IOPS, 128 GB Managed OS Disk Standard : { vmSize: 'Standard_D4s_v5' - minCount: 3 + minCount: 4 maxCount: 20 availabilityZones: [ '1' @@ -419,7 +415,7 @@ var userPoolPresets = { ] osDiskType: 'Managed' osDiskSize: 128 - maxPods: 30 + maxPods: 60 } } diff --git a/docs/pipelines.md b/docs/pipelines.md index 305884eb..a8866734 100644 --- a/docs/pipelines.md +++ b/docs/pipelines.md @@ -6,6 +6,8 @@ A number of CI workflows are leveraged in this repo to test the bicep files to e It's essential to catch as many problems before a single resource is deployed to real infrastructure. There are a lot of tools and techniques that can be leveraged to catch functional or syntactical problems. +![[1]][1] + #### PSRule for Azure An interesting project for performing pre/post validation of Azure Resources against the [Well Architected Framework](https://learn.microsoft.com/en-us/azure/architecture/framework/) is [PSRule for Azure](https://azure.github.io/PSRule.Rules.Azure/). Over [200 rules](https://azure.github.io/PSRule.Rules.Azure/en/baselines/Azure.All/) will be evaluated against the Arm template, ranging from Security configuration to naming conventions. @@ -42,17 +44,21 @@ The fact that a deployment completes successfully is a great sign, however there In this case we are installing software and we are testing the software is installed. -## Solution actions used in this repo +## Github Actions + +This section outlines the key GitHub Actions implemented in this repository to ensure code quality, automate builds, and streamline releases. These actions form the backbone of the continuous integration and deployment pipeline, enhancing our development workflow and maintaining high standards across our project. -### Infra - Test +![[0]][0] + +#### Infra - Test This action will run a Validate Step to ensure any changes to bicep is acceptable. It also will trigger a Standards Check that is non blocking in order to determine if the solution passes the PSRule checks. -### Infra - Build +#### Infra - Build This action ensures that the bicep can build properly. -### Infra - Release +#### Infra - Release The release action will run whenever a release is created to ensure we have a copy of the ARM template from that release that could then be used by other systems as necessary. @@ -60,7 +66,6 @@ The release action will run whenever a release is created to ensure we have a co ```mermaid sequenceDiagram participant Workflow as "GitHub Workflow" - participant Azure as "Azure/login@v2" participant extension1 as "github-tag-action@1.69.0" participant bashCommand as "bash" @@ -69,7 +74,6 @@ sequenceDiagram participant EndBug as "add-and-commit@v9" participant extension3 as "release-action@v1" - Workflow->>Workflow: Manual Trigger Workflow->>Workflow: Code Checkout Workflow->>Azure: Azure Login @@ -82,9 +86,7 @@ sequenceDiagram Workflow->>extension3: Create Release ``` -## Misc actions used in this repo - -### Auto -Documentation Check +#### Auto - Doc This action performs two main functions: @@ -94,10 +96,13 @@ This action performs two main functions: The action is triggered on pull requests that modify markdown files, pushes to the main branch that affect the docs/src directory, and can also be run manually from the Actions tab. This comprehensive approach helps maintain high-quality, current documentation for the project. -### Auto - Greet +#### Auto - Greet This is an auto action for a bot to reply to open issues and open pull requests. -### Auto - Label +#### Auto - Label This is an auto action for a bot to automatically apply labels based on detection of type of code change. + +[0]: src/images/pipeline_1.png "Validation Diagram" +[1]: src/images/pipeline_2.png "Pipeline Diagram" \ No newline at end of file diff --git a/docs/src/architecture.md b/docs/src/architecture.md index 36d940be..7e7f731f 100644 --- a/docs/src/architecture.md +++ b/docs/src/architecture.md @@ -7,13 +7,6 @@ This provides valuable insights into best practices and design considerations of azure workloads. - -**Cost Optimization** - Create a cost-effective solution while balancing security. - -**Security** - Enhance security within a development context, adhering to a zero trust model. - -**Operational Excellence** - Prioritize standards with automation to ensure efficient operations and robust monitoring. - #### Desired State Configuration Bicep is a domain-specific language (DSL) for deploying Azure resources declaratively. It simplifies authoring ARM templates and allows you to define the desired state of your Infrastructure as Code (IaC). Azure Resource Manager (ARM) processes the Bicep file to ensure the Azure environment matches the defined desired state, correcting any drift through redeployment. @@ -103,4 +96,7 @@ A GitOps configuration resides in this Git repository and uses a customized [rep A GitOps approach simplifies the process of deploying and managing software, making it easier to maintain and update, as well as providing a configurable way of leveraging other software configurations by pointing to alternate repositories hosting other configurations. By leveraging this method, we ensure that our deployments can be extended to things that not only include the default software load. +![[1]][1] + [0]: images/architecture.png "Architecture Diagram" +[1]: images/architecture_2.png "Gitops Diagram" \ No newline at end of file diff --git a/docs/src/images/architecture_2.png b/docs/src/images/architecture_2.png new file mode 100644 index 00000000..8a2572ce Binary files /dev/null and b/docs/src/images/architecture_2.png differ diff --git a/docs/src/images/overview_1.png b/docs/src/images/overview_1.png new file mode 100644 index 00000000..4b40e049 Binary files /dev/null and b/docs/src/images/overview_1.png differ diff --git a/docs/src/images/overview_2.png b/docs/src/images/overview_2.png new file mode 100644 index 00000000..caed8754 Binary files /dev/null and b/docs/src/images/overview_2.png differ diff --git a/docs/src/images/pipeline_1.png b/docs/src/images/pipeline_1.png new file mode 100644 index 00000000..57342dda Binary files /dev/null and b/docs/src/images/pipeline_1.png differ diff --git a/docs/src/images/pipeline_2.png b/docs/src/images/pipeline_2.png new file mode 100644 index 00000000..1e862acd Binary files /dev/null and b/docs/src/images/pipeline_2.png differ diff --git a/docs/src/index.md b/docs/src/index.md index b2836551..77a7aa5b 100644 --- a/docs/src/index.md +++ b/docs/src/index.md @@ -9,6 +9,8 @@ OSDU Developer is an open-source solution designed to enable the creation of lig The primary goal for this solution is to provide an environment that can help function within an inner loop process providing faster feedback for developers. This personal environment strives to be user-friendly yet maintain compliance with varying organizational standards. It offers a flexible framework to facilitate deeper exploration of OSDU™ capabilities. +![[0]][0] + Deploying personal instances provide valuable insights into early-stage development and integration processes. It emphasizes transparency, cost-efficiency, and flexibility, empowering developers to engage in essential application and cloud development scenarios. !!! Note @@ -40,6 +42,8 @@ Several use cases illustrate the practical applications for this approach. **Training and Onboarding:** Train new employees on the OSDU™ platform, offering hands-on experience in a controlled environment. +![[1]][1] + ## Features @@ -61,3 +65,5 @@ Several use cases illustrate the practical applications for this approach. Follow the instructions in the "Tutorials" to quickly bring online a personal instance. +[0]: images/overview_1.png "Overview Diagram" +[1]: images/overview_2.png "Use Cases Diagram" \ No newline at end of file diff --git a/docs/src/software.md b/docs/src/software.md index c21ff40f..0229903a 100644 --- a/docs/src/software.md +++ b/docs/src/software.md @@ -1,24 +1,5 @@ # Software Management -In this solution, we utilize a **GitOps** approach for efficient and reliable software management. This method leverages a git repository as the source of truth for defining and updating the software within the cluster and comes with some distinct advantages. - -- **Consistency and Standardization**: Ensures consistent configurations across different environments, minimizing discrepancies. - -- **Audit Trails**: Every change is recorded in Git, providing a clear audit trail for accountability and traceability. - -- **Rollbacks and Recovery**: Allows for easy rollbacks to previous configurations in case of errors or issues. - -- **Enhanced Security**: Changes are reviewed through pull requests, increasing security and promoting collaboration among team members. - -Software definitions are defined in this repository alongside the IaC and employ a customized [repo-per-team](https://fluxcd.io/flux/guides/repository-structure/#repo-per-team) pattern along with customized Helm charts. - -- **Configuration Files**: YAML files that define the desired state of our components and applications. - -- **Charts**: Helm charts used for defining, installing, and upgrading Kubernetes applications. - -Our GitOps approach not only standardizes and secures the software management process but also **simplifies the deployment process**. By making it easier to maintain and update configurations, this approach ensures that deployments are both efficient and flexible. It allows for seamless integration of additional software configurations by pointing to alternate repositories that host these configurations. This extensibility enables our deployments to include not just the default software load but also any additional components required by our architecture. - - ### Stamp Layout In our software architecture design, we define two primary software Kustomizations that describe a **stamp**. A Kustomization is a Flux resource representing a set of defined manifests that Flux should reconcile to the cluster, with dependencies between them. Structuring our Kustomizations this way ensures clarity and separation of concerns, making it easier to manage and organize both components and applications. @@ -174,3 +155,78 @@ flowchart TD │ └── source.yaml ``` +__OSDU Core Structure__ + +The OSDU Core application is organized to facilitate the management of the OSDU core platform services. Below is the layout: + +```mermaid +flowchart TD + base("base") + partition("partition") + partition_init("partition-init") + entitlements("entitlements") + entitlements_init("entitlements-init") + legal("legal") + indexer("indexer") + indexer_queue("indexer-queue") + user_init("user-init") + schema("schema") + schema_init("schema-init") + storage("storage") + file("file") + search("search") + + base-->partition + partition-->entitlements + partition-->partition_init + entitlements-->entitlements_init + entitlements_init-->user_init + partition-->legal + legal-->indexer + legal-->indexer_queue + legal-->schema + schema-->schema_init + indexer_queue-->storage + indexer_queue-->file + indexer_queue-->search +``` + +```bash +── osdu-core +   ├── base.yaml +   ├── entitlements.yaml +   ├── file.yaml +   ├── indexer.yaml +   ├── legal.yaml +   ├── namespace.yaml +   ├── partition.yaml +   ├── schema.yaml +   ├── search.yaml +   ├── storage.yaml +   └── user-init.yaml +``` + +__OSDU Reference Structure__ + +The OSDU Reference application is organized to facilitate the management of the OSDU reference platform services. Below is the layout: + +```mermaid +flowchart TD + base("base") + unit("unit") + crs-catalog("crs-catalog") + crs-conversion("crs-conversion") + + base-->unit + base-->crs-catalog + base-->crs-conversion +``` + +```bash +── osdu-reference +   ├── base.yaml +   ├── crs-catalog.yaml +   ├── crs-conversion.yaml +   ├── namespace.yaml +   └── unit.yaml +``` \ No newline at end of file diff --git a/software/applications/osdu-core/README.md b/software/applications/osdu-core/README.md index d62970c7..9ba930d2 100644 --- a/software/applications/osdu-core/README.md +++ b/software/applications/osdu-core/README.md @@ -21,16 +21,13 @@ flowchart TD partition-->entitlements partition-->partition_init entitlements-->entitlements_init - partition_init-->entitlements_init - entitlements_init-->legal + entitlements_init-->user_init + partition-->legal legal-->indexer legal-->indexer_queue - entitlements_init-->user_init - indexer-->schema - indexer_queue-->schema + legal-->schema schema-->schema_init - schema-->storage - schema-->file - storage-->search - file-->search + indexer_queue-->storage + indexer_queue-->file + indexer_queue-->search ``` \ No newline at end of file diff --git a/software/applications/osdu-core/file.yaml b/software/applications/osdu-core/file.yaml index 7f66f07b..cf5229d4 100644 --- a/software/applications/osdu-core/file.yaml +++ b/software/applications/osdu-core/file.yaml @@ -7,6 +7,9 @@ metadata: annotations: clusterconfig.azure.com/use-managed-source: "true" spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core targetNamespace: osdu-core chart: spec: diff --git a/software/applications/osdu-core/indexer.yaml b/software/applications/osdu-core/indexer.yaml index 93aedb66..a3c544d7 100644 --- a/software/applications/osdu-core/indexer.yaml +++ b/software/applications/osdu-core/indexer.yaml @@ -7,6 +7,10 @@ metadata: annotations: clusterconfig.azure.com/use-managed-source: "true" spec: + dependsOn: + - name: osdu-legal + namespace: osdu-core + targetNamespace: osdu-core chart: spec: chart: ./charts/osdu-developer-service @@ -19,7 +23,6 @@ spec: install: remediation: retries: 3 - targetNamespace: osdu-core valuesFrom: - kind: ConfigMap name: configmap-software @@ -147,6 +150,10 @@ metadata: annotations: clusterconfig.azure.com/use-managed-source: "true" spec: + dependsOn: + - name: osdu-legal + namespace: osdu-core + targetNamespace: osdu-core chart: spec: chart: ./charts/osdu-developer-service @@ -159,7 +166,6 @@ spec: install: remediation: retries: 3 - targetNamespace: osdu-core valuesFrom: - kind: ConfigMap name: configmap-software diff --git a/software/applications/osdu-core/legal.yaml b/software/applications/osdu-core/legal.yaml index eb78c3bd..d1e46741 100644 --- a/software/applications/osdu-core/legal.yaml +++ b/software/applications/osdu-core/legal.yaml @@ -7,6 +7,9 @@ metadata: annotations: clusterconfig.azure.com/use-managed-source: "true" spec: + dependsOn: + - name: osdu-partition + namespace: osdu-core targetNamespace: osdu-core chart: spec: diff --git a/software/applications/osdu-core/search.yaml b/software/applications/osdu-core/search.yaml index 885ee3ba..be1fa8af 100644 --- a/software/applications/osdu-core/search.yaml +++ b/software/applications/osdu-core/search.yaml @@ -7,6 +7,9 @@ metadata: annotations: clusterconfig.azure.com/use-managed-source: "true" spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core targetNamespace: osdu-core chart: spec: diff --git a/software/applications/osdu-core/storage.yaml b/software/applications/osdu-core/storage.yaml index 5d2dab7c..6572a454 100644 --- a/software/applications/osdu-core/storage.yaml +++ b/software/applications/osdu-core/storage.yaml @@ -7,6 +7,9 @@ metadata: annotations: clusterconfig.azure.com/use-managed-source: "true" spec: + dependsOn: + - name: osdu-indexer-queue + namespace: osdu-core chart: spec: chart: ./charts/osdu-developer-service