Merge pull request #142 from DataCloud-project/resource-prediction

Resource prediction

charts/simpipe/templates/deployment-controller.yaml

@@ -44,8 +44,10 @@ spec:
                   key: minioSecretKey
             - name: JWT_DEV_MODE
               value: "true"
-            - name: MINIO_URL
-              value: {{ .Values.controller.minio.url | quote }}
+            - name: MINIO_PUBLIC_URL
+              value: {{ .Values.controller.minio.publicUrl | quote }}
+            - name: MINIO_INTERNAL_URL
+              value: {{ .Values.controller.minio.internalUrl | quote }}
             - name: MINIO_BUCKET_NAME
               value: {{ .Values.controller.minio.bucketName | quote }}
             - name: MINIO_REGION

charts/simpipe/values.yaml

@@ -22,7 +22,8 @@ controller:
 
   minio:
     # Configured by default for the localhost port forwarding in forwarding.py
-    url: http://localhost:8085
+    publicUrl: http://localhost:8085
+    internalUrl: http://simpipe-minio.default.svc.cluster.local:9000
     accessKey: simpipe
     secretKey: simpipe1234
     bucketName: artifacts

controller/src/argo/dry-runs.ts

@@ -5,7 +5,7 @@ import {
 import { DryRunNodePhase, DryRunNodeType, DryRunPhase } from '../server/schema.js';
 import getPodName from './get-pod-name.js';
 import type {
-  DryRun, DryRunNode, DryRunNodeArtifact, DryRunNodePod,
+  Artifact, DryRun, DryRunNode, DryRunNodePod,
 } from '../server/schema.js';
 import type { ArgoClientActionNames, ArgoNode, ArgoWorkflow } from './argo-client.js';
 import type ArgoWorkflowClient from './argo-client.js';
@@ -125,8 +125,8 @@ export function convertArgoWorkflowNode(node: ArgoNode, argoWorkflow: ArgoWorkfl
   const type = convertArgoNodeType(node.type);
 
   let podName: string | undefined;
-  let inputArtifacts: DryRunNodeArtifact[] | undefined;
-  let outputArtifacts: DryRunNodeArtifact[] | undefined;
+  let inputArtifacts: Artifact[] | undefined;
+  let outputArtifacts: Artifact[] | undefined;
 
   if (type === DryRunNodeType.Pod) {
     podName = getPodName(node, argoWorkflow);

controller/src/config.ts

@@ -16,7 +16,8 @@ export const minioAccessKey = process.env.MINIO_ACCESS_KEY;
 export const minioSecretKey = process.env.MINIO_SECRET_KEY;
 export const minioBucketName = process.env.MINIO_BUCKET_NAME ?? 'artifacts';
 export const minioRegion = process.env.MINIO_REGION ?? 'no-region';
-export const minioUrl = process.env.MINIO_URL ?? 'http://localhost:8085';
+export const minioPublicUrl = process.env.MINIO_PUBLIC_URL ?? 'http://localhost:8085';
+export const minioInternalUrl = process.env.MINIO_INTERNAL_URL ?? minioPublicUrl;
 
 // Argo client
 export const argoClientEndpoint = process.env.ARGO_CLIENT_ENDPOINT ?? 'http://localhost:8084/';

controller/src/minio/minio.ts

@@ -4,9 +4,10 @@ import { URL } from 'node:url';
 import {
   minioAccessKey,
   minioBucketName,
+  minioInternalUrl,
+  minioPublicUrl,
   minioRegion,
   minioSecretKey,
-  minioUrl,
 } from '../config.js';
 
 if (!minioAccessKey) {
@@ -16,23 +17,37 @@ if (!minioSecretKey) {
   throw new Error('MINIO_SECRET_KEY is not set');
 }
 
-const parsedUrl = new URL(minioUrl);
-const minioClient = new MinioClient({
-  endPoint: parsedUrl.hostname,
-  port: Number.parseInt(parsedUrl.port, 10) ?? (parsedUrl.protocol === 'https:' ? 443 : 80),
-  useSSL: parsedUrl.protocol === 'https:',
+const parsedMinioPublicUrl = new URL(minioPublicUrl);
+const minioPublicClient = new MinioClient({
+  endPoint: parsedMinioPublicUrl.hostname,
+  port: Number.parseInt(parsedMinioPublicUrl.port, 10) ?? (parsedMinioPublicUrl.protocol === 'https:' ? 443 : 80),
+  useSSL: parsedMinioPublicUrl.protocol === 'https:',
+  accessKey: minioAccessKey,
+  secretKey: minioSecretKey,
+  region: minioRegion,
+});
+const parsedMinioInternalUrl = new URL(minioInternalUrl);
+const minioInternalClient = new MinioClient({
+  endPoint: parsedMinioInternalUrl.hostname,
+  port: Number.parseInt(parsedMinioInternalUrl.port, 10) ?? (parsedMinioInternalUrl.protocol === 'https:' ? 443 : 80),
+  useSSL: parsedMinioInternalUrl.protocol === 'https:',
   accessKey: minioAccessKey,
   secretKey: minioSecretKey,
   region: minioRegion,
 });
 
 export async function computePresignedPutUrl(objectName: string): Promise<string> {
   const expire = 60 * 60 * 24 * 7; // 7 days
-  return await minioClient.presignedPutObject(minioBucketName, objectName, expire);
+  return await minioPublicClient.presignedPutObject(minioBucketName, objectName, expire);
 }
 
 export async function computePresignedGetUrl(objectName: string): Promise<string> {
   // One hour
   const expire = 3600;
-  return await minioClient.presignedGetObject(minioBucketName, objectName, expire);
+  return await minioPublicClient.presignedGetObject(minioBucketName, objectName, expire);
+}
+
+export async function getObjectSize(objectName: string): Promise<number> {
+  const stat = await minioInternalClient.statObject(minioBucketName, objectName);
+  return stat.size;
 }

controller/src/server/resolvers.ts

@@ -26,18 +26,18 @@ import { SIMPIPE_PROJECT_LABEL } from '../k8s/label.js';
 import {
   createProject, deleteProject, getProject, projects, renameProject,
 } from '../k8s/projects.js';
-import { computePresignedGetUrl, computePresignedPutUrl } from '../minio/minio.js';
+import { computePresignedGetUrl, computePresignedPutUrl, getObjectSize } from '../minio/minio.js';
 import { assertPrometheusIsHealthy } from '../prometheus/prometheus.js';
 import queryPrometheusResolver from '../prometheus/query-prometheus-resolver.js';
 import { NotFoundError, PingError } from './apollo-errors.js';
 import type { ArgoWorkflow, ArgoWorkflowTemplate } from '../argo/argo-client.js';
 import type ArgoWorkflowClient from '../argo/argo-client.js';
 import type K8sClient from '../k8s/k8s-client.js';
 import type {
+  Artifact,
   DryRun,
   DryRunNode,
   DryRunNodeArgs as DryRunNodeArguments,
-  DryRunNodeArtifact,
   DryRunNodeMetrics,
   DryRunNodeMetricsCpuSystemSecondsTotalArgs as DryRunNodeMetricsCpuSystemSecondsTotalArguments,
   DryRunNodePod, DryRunNodePodLogArgs as DryRunNodePodLogArguments,
@@ -160,6 +160,13 @@ const resolvers = {
       const { sub } = user;
       return await getWorkflowTemplate(name, argoClient, sub);
     },
+    async artifacts(
+      _p: EmptyParent, _a: EmptyArguments, context: AuthenticatedContext,
+    ): Promise<Query['artifacts']> {
+      // eslint-disable-next-line @typescript-eslint/no-unused-vars
+      const { argoClient } = context;
+      return [];
+    },
   } as Required<QueryResolvers<AuthenticatedContext, EmptyParent>>,
   Mutation: {
     async createDryRun(
@@ -547,16 +554,25 @@ const resolvers = {
     threadsMax: queryPrometheusResolver.bind(undefined, 'simpipe_threads_max', 'main'),
     ulimitsSoft: queryPrometheusResolver.bind(undefined, 'simpipe_ulimits_soft', 'main'),
   },
-  DryRunNodeArtifact: {
+  Artifact: {
     async url(
-      dryRunNodeArtifact: DryRunNodeArtifact,
-    ): Promise<DryRunNodeArtifact['url']> {
-      const { key } = dryRunNodeArtifact;
+      artifact: Artifact,
+    ): Promise<Artifact['url']> {
+      const { key } = artifact;
       if (!key) {
         return undefined;
       }
       return await computePresignedGetUrl(key);
     },
+    async size(
+      artifact: Artifact,
+    ): Promise<Artifact['size']> {
+      const { key } = artifact;
+      if (!key) {
+        return undefined;
+      }
+      return await getObjectSize(key);
+    },
   },
   WorkflowTemplate: {
     async project(

controller/src/server/schema.graphql

@@ -39,6 +39,9 @@ type Query {
   workflowTemplate(
     """ Name of the Argo workflow template to be fetched """
     name: String!): WorkflowTemplate @auth
+
+  """ List of all the artifacts """
+  artifacts: [Artifact!]! @auth
 }
 
 """ Project is the high level concept for a specific pipeline in SIM-PIPE. It contains all the previous dry runs of the pipeline and its results. """
@@ -229,10 +232,10 @@ type DryRunNodePod implements DryRunNode {
   log(maxLines: Int, grep: String, sinceSeconds: Int, sinceTime: Int): [String!]
 
   """ Input artifacts of the node. """
-  inputArtifacts: [DryRunNodeArtifact!]
+  inputArtifacts: [Artifact!]
 
   """ Output artifacts of the node. """
-  outputArtifacts: [DryRunNodeArtifact!]
+  outputArtifacts: [Artifact!]
 }
 
 type DryRunNodeMisc implements DryRunNode {
@@ -301,13 +304,15 @@ type DryRunNodeMetrics {
 }
 
 """ Contains information about files produced during a dry run execution """
-type DryRunNodeArtifact {
+type Artifact {
   """ The artifact name """
   name: String!
   """ The artifact path """
   key: String
   """ URL to download the artifact using an HTTP GET."""
   url: String
+  """ The artifact size """
+  size: Int
 }
 
 """

controller/src/server/schema.ts

@@ -31,6 +31,19 @@ export type Scalars = {
   TimeStamp: { input: number; output: number; }
 };
 
+/**  Contains information about files produced during a dry run execution  */
+export type Artifact = {
+  __typename?: 'Artifact';
+  /**  The artifact path  */
+  key?: Maybe<Scalars['String']['output']>;
+  /**  The artifact name  */
+  name: Scalars['String']['output'];
+  /**  The artifact size  */
+  size?: Maybe<Scalars['Int']['output']>;
+  /**  URL to download the artifact using an HTTP GET. */
+  url?: Maybe<Scalars['String']['output']>;
+};
+
 /**  The input data to create a new dry run  */
 export type CreateDryRunInput = {
   /**
@@ -139,17 +152,6 @@ export type DryRunNode = {
   type: DryRunNodeType;
 };
 
-/**  Contains information about files produced during a dry run execution  */
-export type DryRunNodeArtifact = {
-  __typename?: 'DryRunNodeArtifact';
-  /**  The artifact path  */
-  key?: Maybe<Scalars['String']['output']>;
-  /**  The artifact name  */
-  name: Scalars['String']['output'];
-  /**  URL to download the artifact using an HTTP GET. */
-  url?: Maybe<Scalars['String']['output']>;
-};
-
 /**  Prometheus metrics for the node.  */
 export type DryRunNodeMetrics = {
   __typename?: 'DryRunNodeMetrics';
@@ -569,14 +571,14 @@ export type DryRunNodePod = DryRunNode & {
   finishedAt?: Maybe<Scalars['String']['output']>;
   id: Scalars['String']['output'];
   /**  Input artifacts of the node.  */
-  inputArtifacts?: Maybe<Array<DryRunNodeArtifact>>;
+  inputArtifacts?: Maybe<Array<Artifact>>;
   /**  The logs of the node.  */
   log?: Maybe<Array<Scalars['String']['output']>>;
   /**  The name of the pod.  */
   metrics: DryRunNodeMetrics;
   name: Scalars['String']['output'];
   /**  Output artifacts of the node.  */
-  outputArtifacts?: Maybe<Array<DryRunNodeArtifact>>;
+  outputArtifacts?: Maybe<Array<Artifact>>;
   phase: DryRunNodePhase;
   /**  The name of the pod.  */
   podName: Scalars['String']['output'];
@@ -823,6 +825,8 @@ export type PrometheusSample = {
 
 export type Query = {
   __typename?: 'Query';
+  /**  List of all the artifacts  */
+  artifacts: Array<Artifact>;
   /**  List of all docker registry credentials  */
   dockerRegistryCredentials: Array<DockerRegistryCredential>;
   /**  Get a dry run by ID  */
@@ -952,6 +956,7 @@ export type ResolversInterfaceTypes<RefType extends Record<string, unknown>> = {
 export type ResolversTypes = {
   ArgoWorkflow: ResolverTypeWrapper<Scalars['ArgoWorkflow']['output']>;
   ArgoWorkflowTemplate: ResolverTypeWrapper<Scalars['ArgoWorkflowTemplate']['output']>;
+  Artifact: ResolverTypeWrapper<Artifact>;
   Boolean: ResolverTypeWrapper<Scalars['Boolean']['output']>;
   CreateDryRunInput: CreateDryRunInput;
   CreateProjectInput: CreateProjectInput;
@@ -960,7 +965,6 @@ export type ResolversTypes = {
   DockerRegistryCredentialInput: DockerRegistryCredentialInput;
   DryRun: ResolverTypeWrapper<DryRun>;
   DryRunNode: ResolverTypeWrapper<ResolversInterfaceTypes<ResolversTypes>['DryRunNode']>;
-  DryRunNodeArtifact: ResolverTypeWrapper<DryRunNodeArtifact>;
   DryRunNodeMetrics: ResolverTypeWrapper<DryRunNodeMetrics>;
   DryRunNodeMisc: ResolverTypeWrapper<DryRunNodeMisc>;
   DryRunNodePhase: DryRunNodePhase;
@@ -985,6 +989,7 @@ export type ResolversTypes = {
 export type ResolversParentTypes = {
   ArgoWorkflow: Scalars['ArgoWorkflow']['output'];
   ArgoWorkflowTemplate: Scalars['ArgoWorkflowTemplate']['output'];
+  Artifact: Artifact;
   Boolean: Scalars['Boolean']['output'];
   CreateDryRunInput: CreateDryRunInput;
   CreateProjectInput: CreateProjectInput;
@@ -993,7 +998,6 @@ export type ResolversParentTypes = {
   DockerRegistryCredentialInput: DockerRegistryCredentialInput;
   DryRun: DryRun;
   DryRunNode: ResolversInterfaceTypes<ResolversParentTypes>['DryRunNode'];
-  DryRunNodeArtifact: DryRunNodeArtifact;
   DryRunNodeMetrics: DryRunNodeMetrics;
   DryRunNodeMisc: DryRunNodeMisc;
   DryRunNodePod: DryRunNodePod;
@@ -1019,6 +1023,14 @@ export interface ArgoWorkflowTemplateScalarConfig extends GraphQLScalarTypeConfi
   name: 'ArgoWorkflowTemplate';
 }
 
+export type ArtifactResolvers<ContextType = any, ParentType extends ResolversParentTypes['Artifact'] = ResolversParentTypes['Artifact']> = {
+  key?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
+  size?: Resolver<Maybe<ResolversTypes['Int']>, ParentType, ContextType>;
+  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
+  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
+};
+
 export type DockerRegistryCredentialResolvers<ContextType = any, ParentType extends ResolversParentTypes['DockerRegistryCredential'] = ResolversParentTypes['DockerRegistryCredential']> = {
   name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   server?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
@@ -1052,13 +1064,6 @@ export type DryRunNodeResolvers<ContextType = any, ParentType extends ResolversP
   type?: Resolver<ResolversTypes['DryRunNodeType'], ParentType, ContextType>;
 };
 
-export type DryRunNodeArtifactResolvers<ContextType = any, ParentType extends ResolversParentTypes['DryRunNodeArtifact'] = ResolversParentTypes['DryRunNodeArtifact']> = {
-  key?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  url?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
-  __isTypeOf?: IsTypeOfResolverFn<ParentType, ContextType>;
-};
-
 export type DryRunNodeMetricsResolvers<ContextType = any, ParentType extends ResolversParentTypes['DryRunNodeMetrics'] = ResolversParentTypes['DryRunNodeMetrics']> = {
   cpuSystemSecondsTotal?: Resolver<Maybe<Array<ResolversTypes['PrometheusSample']>>, ParentType, ContextType, Partial<DryRunNodeMetricsCpuSystemSecondsTotalArgs>>;
   cpuUsageSecondsTotal?: Resolver<Maybe<Array<ResolversTypes['PrometheusSample']>>, ParentType, ContextType, Partial<DryRunNodeMetricsCpuUsageSecondsTotalArgs>>;
@@ -1126,11 +1131,11 @@ export type DryRunNodePodResolvers<ContextType = any, ParentType extends Resolve
   exitCode?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   finishedAt?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
   id?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  inputArtifacts?: Resolver<Maybe<Array<ResolversTypes['DryRunNodeArtifact']>>, ParentType, ContextType>;
+  inputArtifacts?: Resolver<Maybe<Array<ResolversTypes['Artifact']>>, ParentType, ContextType>;
   log?: Resolver<Maybe<Array<ResolversTypes['String']>>, ParentType, ContextType, Partial<DryRunNodePodLogArgs>>;
   metrics?: Resolver<ResolversTypes['DryRunNodeMetrics'], ParentType, ContextType>;
   name?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
-  outputArtifacts?: Resolver<Maybe<Array<ResolversTypes['DryRunNodeArtifact']>>, ParentType, ContextType>;
+  outputArtifacts?: Resolver<Maybe<Array<ResolversTypes['Artifact']>>, ParentType, ContextType>;
   phase?: Resolver<ResolversTypes['DryRunNodePhase'], ParentType, ContextType>;
   podName?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
   progress?: Resolver<Maybe<ResolversTypes['String']>, ParentType, ContextType>;
@@ -1199,6 +1204,7 @@ export interface PrometheusStringNumberScalarConfig extends GraphQLScalarTypeCon
 }
 
 export type QueryResolvers<ContextType = any, ParentType extends ResolversParentTypes['Query'] = ResolversParentTypes['Query']> = {
+  artifacts?: Resolver<Array<ResolversTypes['Artifact']>, ParentType, ContextType>;
   dockerRegistryCredentials?: Resolver<Array<ResolversTypes['DockerRegistryCredential']>, ParentType, ContextType>;
   dryRun?: Resolver<ResolversTypes['DryRun'], ParentType, ContextType, RequireFields<QueryDryRunArgs, 'dryRunId'>>;
   ping?: Resolver<ResolversTypes['String'], ParentType, ContextType>;
@@ -1222,10 +1228,10 @@ export type WorkflowTemplateResolvers<ContextType = any, ParentType extends Reso
 export type Resolvers<ContextType = any> = {
   ArgoWorkflow?: GraphQLScalarType;
   ArgoWorkflowTemplate?: GraphQLScalarType;
+  Artifact?: ArtifactResolvers<ContextType>;
   DockerRegistryCredential?: DockerRegistryCredentialResolvers<ContextType>;
   DryRun?: DryRunResolvers<ContextType>;
   DryRunNode?: DryRunNodeResolvers<ContextType>;
-  DryRunNodeArtifact?: DryRunNodeArtifactResolvers<ContextType>;
   DryRunNodeMetrics?: DryRunNodeMetricsResolvers<ContextType>;
   DryRunNodeMisc?: DryRunNodeMiscResolvers<ContextType>;
   DryRunNodePod?: DryRunNodePodResolvers<ContextType>;