Merge pull request #1308 from Genez-io/add-backend-environment

Add backend.environment to the configuration file

src/commands/deploy/genezio.ts

@@ -548,7 +548,14 @@ export async function deployClasses(
         const cwd = projectConfiguration.workspace?.backend
             ? path.resolve(projectConfiguration.workspace.backend)
             : process.cwd();
-        await uploadEnvVarsFromFile(options.env, projectId, projectEnvId, cwd);
+        await uploadEnvVarsFromFile(
+            options.env,
+            projectId,
+            projectEnvId,
+            cwd,
+            options.stage || "prod",
+            configuration,
+        );
 
         return {
             projectId: projectId,

src/commands/deploy/nextjs/deploy.ts

@@ -98,6 +98,8 @@ export async function nextJsDeploy(options: GenezioDeployOptions) {
             deploymentResult.projectId,
             deploymentResult.projectEnvId,
             process.cwd(),
+            options.stage || "prod",
+            genezioConfig,
         ),
     ]);
 

src/commands/deploy/utils.ts

@@ -25,8 +25,11 @@ import {
     detectEnvironmentVariablesFile,
     findAnEnvFile,
     getUnsetEnvironmentVariables,
+    parseConfigurationVariable,
     promptToConfirmSettingEnvironmentVariables,
+    resolveEnvironmentVariable,
 } from "../../utils/environmentVariables.js";
+import { EnvironmentVariable } from "../../models/environmentVariables.js";
 
 export async function getOrCreateEmptyProject(
     projectName: string,
@@ -163,6 +166,8 @@ export async function uploadEnvVarsFromFile(
     projectId: string,
     projectEnvId: string,
     cwd: string,
+    stage: string,
+    configuration: YamlProjectConfiguration,
 ) {
     if (envPath) {
         const envFile = path.join(process.cwd(), envPath);
@@ -182,8 +187,9 @@ export async function uploadEnvVarsFromFile(
             // Upload environment variables to the project
             await setEnvironmentVariables(projectId, projectEnvId, envVars)
                 .then(async () => {
+                    const envVarKeys = envVars.map((envVar) => envVar.name);
                     log.info(
-                        `The following environment variables ${envVars.join(", ")} were uploaded to the project successfully.`,
+                        `The following environment variables ${envVarKeys.join(", ")} were uploaded to the project successfully.`,
                     );
                     await GenezioTelemetry.sendEvent({
                         eventType: TelemetryEventTypes.GENEZIO_DEPLOY_LOAD_ENV_VARS,
@@ -202,12 +208,11 @@ export async function uploadEnvVarsFromFile(
                     });
                 });
         }
-        return;
     }
 
     // This is best effort, we should encourage the user to use `--env <envFile>` to set the correct env file path.
     // Search for possible .env files in the project directory and use the first
-    const envFile = await findAnEnvFile(cwd);
+    const envFile = envPath ? path.join(process.cwd(), envPath) : await findAnEnvFile(cwd);
 
     if (!envFile) {
         return;
@@ -220,6 +225,46 @@ export async function uploadEnvVarsFromFile(
         projectEnvId,
     );
 
+    const environment = configuration.backend?.environment;
+    if (environment) {
+        const unsetEnvVarKeys = await getUnsetEnvironmentVariables(
+            Object.keys(environment),
+            projectId,
+            projectEnvId,
+        );
+
+        const environmentVariablesToBePushed: EnvironmentVariable[] = (
+            await Promise.all(
+                unsetEnvVarKeys.map(async (envVarKey) => {
+                    const variable = await parseConfigurationVariable(environment[envVarKey]);
+                    const resolvedVariable = await resolveEnvironmentVariable(
+                        configuration,
+                        variable,
+                        envVarKey,
+                        envFile,
+                        stage,
+                    );
+                    if (!resolvedVariable) {
+                        return undefined;
+                    }
+                    return resolvedVariable;
+                }),
+            )
+        ).filter((item): item is EnvironmentVariable => item !== undefined);
+
+        if (environmentVariablesToBePushed.length > 0) {
+            debugLogger.debug(
+                `Uploading environment variables ${JSON.stringify(environmentVariablesToBePushed)} from ${envFile} to project ${projectId}`,
+            );
+            await setEnvironmentVariables(projectId, projectEnvId, environmentVariablesToBePushed);
+            debugLogger.debug(
+                `Environment variables uploaded to project ${projectId} successfully.`,
+            );
+        }
+
+        return;
+    }
+
     if (
         !process.env["CI"] &&
         missingEnvVars.length > 0 &&
@@ -232,7 +277,9 @@ export async function uploadEnvVarsFromFile(
             await promptToConfirmSettingEnvironmentVariables(missingEnvVars);
 
         if (!confirmSettingEnvVars) {
-            log.info("Skipping environment variables upload.");
+            log.info(
+                `Skipping environment variables upload. You can set them later by navigation to the dashboard ${DASHBOARD_URL}`,
+            );
         } else {
             const environmentVariablesToBePushed = envVars.filter((envVar: { name: string }) =>
                 missingEnvVars.includes(envVar.name),
@@ -246,8 +293,9 @@ export async function uploadEnvVarsFromFile(
                 projectEnvId,
                 environmentVariablesToBePushed,
             ).then(async () => {
+                const envVarKeys = envVars.map((envVar) => envVar.name);
                 log.info(
-                    `The following environment variables ${envVars.join(", ")} were uploaded to the project successfully.`,
+                    `The following environment variables ${envVarKeys.join(", ")} were uploaded to the project successfully.`,
                 );
                 await GenezioTelemetry.sendEvent({
                     eventType: TelemetryEventTypes.GENEZIO_DEPLOY_LOAD_ENV_VARS,

src/projectConfiguration/yaml/v2.ts

@@ -39,6 +39,8 @@ function parseGenezioConfig(config: unknown) {
 
     const scriptSchema = zod.array(zod.string()).or(zod.string()).optional();
 
+    const environmentSchema = zod.record(zod.string(), zod.string());
+
     const methodSchema = zod
         .object({
             name: zod.string(),
@@ -114,6 +116,7 @@ function parseGenezioConfig(config: unknown) {
     const backendSchema = zod.object({
         path: zod.string(),
         language: languageSchema,
+        environment: environmentSchema.optional(),
         scripts: zod
             .object({
                 deploy: scriptSchema,
@@ -134,7 +137,7 @@ function parseGenezioConfig(config: unknown) {
             .optional(),
         subdomain: zod.string().optional(),
         publish: zod.string().optional(),
-        environment: zod.record(zod.string(), zod.string()).optional(),
+        environment: environmentSchema.optional(),
         scripts: zod
             .object({
                 build: scriptSchema,

src/utils/environmentVariables.ts

@@ -1,12 +1,147 @@
 import inquirer from "inquirer";
-import { fileExists } from "./file.js";
+import { fileExists, readEnvironmentVariablesFile } from "./file.js";
 import { getEnvironmentVariables } from "../requests/getEnvironmentVariables.js";
 import path from "path";
+import { resolveConfigurationVariable } from "./scripts.js";
+import { debugLogger, log } from "./logging.js";
+import { YamlProjectConfiguration } from "../projectConfiguration/yaml/v2.js";
+import { EnvironmentVariable } from "../models/environmentVariables.js";
 
+export type ConfigurationVariable =
+    | {
+          path: string;
+          field: string;
+      }
+    | {
+          key: string;
+      }
+    | {
+          value: string;
+      };
+
+/**
+ * Parses a configuration variable string to extract the path and field.
+ *
+ * @param rawValue The raw string value of the configuration variable.
+ * @returns An object containing the path and field, or the key, or the value.
+ *
+ * @example
+ * parseConfigurationVariable("${{ backend.functions.<function-name>.url }}");
+ * // Returns { path: "backend.functions.<function-name>", field: "url" }
+ *
+ * @example
+ * parseConfigurationVariable("${{ env.MY_ENV_VAR }}");
+ * // Returns { key: "MY_ENV_VAR" }
+ *
+ * @example
+ * parseConfigurationVariable("my-value");
+ * // Returns { value: "my-value" }
+ */
+export async function parseConfigurationVariable(rawValue: string): Promise<ConfigurationVariable> {
+    const prefix = "${{";
+    const suffix = "}}";
+
+    const sanitizeVariable = (variable: string): string =>
+        variable.slice(prefix.length, -suffix.length).replace(/\s/g, "");
+
+    // Format: ${{ env.<variable> }}
+    const regexEnv = /\$\{\{[ ]*env\.[ a-zA-Z0-9-._]+\}\}/;
+    const matchEnv = rawValue.match(regexEnv);
+    if (matchEnv) {
+        const variable = sanitizeVariable(matchEnv[0]);
+        // Split the string at the first period to get <variable>
+        const firstDotIndex = variable.indexOf(".");
+        const key = variable.substring(firstDotIndex + 1);
+        return { key };
+    }
+
+    // Format: ${{ backend.functions.<function-name>.url }}
+    const regex = /\$\{\{[ a-zA-Z0-9-._]+\}\}/;
+    const match = rawValue.match(regex);
+    if (match) {
+        // Sanitize the variable
+        const variable = sanitizeVariable(match[0]);
+        // Split the string at the last period to get the path `backend.functions.<function-name>` and field `url`
+        const lastDotIndex = variable.lastIndexOf(".");
+        const path = variable.substring(0, lastDotIndex);
+        const field = variable.substring(lastDotIndex + 1);
+        return { path, field };
+    }
+
+    return { value: rawValue };
+}
+
+export async function resolveEnvironmentVariable(
+    configuration: YamlProjectConfiguration,
+    variable: ConfigurationVariable,
+    envVarKey: string,
+    envFile: string,
+    stage: string,
+): Promise<EnvironmentVariable | undefined> {
+    if ("path" in variable && "field" in variable) {
+        debugLogger.debug(
+            `Resolving configuration variable for environment variable ${envVarKey} for <path>.<field> format`,
+        );
+        const resolvedValue = await resolveConfigurationVariable(
+            configuration,
+            stage,
+            variable.path,
+            variable.field,
+        );
+        return {
+            name: envVarKey,
+            value: resolvedValue,
+        };
+    } else if ("key" in variable) {
+        debugLogger.debug(
+            `Resolving environment variable from configuration file for ${envVarKey} for env.<key> format`,
+        );
+        const envVar = (await readEnvironmentVariablesFile(envFile)).find(
+            (envVar) => envVar.name === variable.key,
+        );
+        if (envVar) {
+            return {
+                name: envVarKey,
+                value: envVar.value,
+            };
+        }
+
+        if (process.env[envVarKey]) {
+            return {
+                name: envVarKey,
+                value: process.env[envVarKey],
+            };
+        }
+
+        log.warn(`Environment variable ${envVarKey} is missing from the ${envFile} file.`);
+    } else if ("value" in variable) {
+        debugLogger.debug(
+            `Resolving environment variable from configuration file for ${envVarKey} for cleartext`,
+        );
+        return {
+            name: envVarKey,
+            value: variable.value,
+        };
+    }
+    return undefined;
+}
+
+/**
+ * Detects if an environment variables file exists at the given path.
+ * @param path The path to the environment variables file.
+ *
+ * @returns A boolean indicating if the file exists.
+ */
 export async function detectEnvironmentVariablesFile(path: string) {
     return await fileExists(path);
 }
 
+/**
+ * Prompts the user to confirm setting the detected environment variables.
+ *
+ * @param envVars The list of environment variables to set.
+ * @returns A boolean indicating if the user confirmed setting the environment variables.
+ */
 export async function promptToConfirmSettingEnvironmentVariables(envVars: string[]) {
     const { confirmSetEnvVars }: { confirmSetEnvVars: boolean } = await inquirer.prompt([
         {
@@ -24,6 +159,14 @@ export async function promptToConfirmSettingEnvironmentVariables(envVars: string
     return true;
 }
 
+/**
+ * Gets the list of environment variables that were found locally but not set remotely.
+ *
+ * @param local The list of environment variables found locally.
+ * @param projectId The project ID.
+ * @param projectEnvId The project environment ID.
+ * @returns The list of environment variables that were found locally but not set remotely.
+ */
 export async function getUnsetEnvironmentVariables(
     local: string[],
     projectId: string,
@@ -38,7 +181,13 @@ export async function getUnsetEnvironmentVariables(
     return missingEnvVars;
 }
 
+/**
+ * Finds an environment variables file in the given directory.
+ * @param cwd The directory to search for the environment variables file.
+ * @returns The path to the environment variables file.
+ */
 export async function findAnEnvFile(cwd: string): Promise<string | undefined> {
+    // These are the most common locations for the .env file
     const possibleEnvFilePath = ["server/.env", ".env"];
 
     for (const envFilePath of possibleEnvFilePath) {

tests/utils/environmentVariables.test.ts

@@ -0,0 +1,40 @@
+import { describe, it, expect } from "vitest";
+import { parseConfigurationVariable } from "../../src/utils/environmentVariables";
+
+describe("parseConfigurationVariable", () => {
+    it("should parse env variable correctly", async () => {
+        const rawValue = "${{ env.MY_ENV_VAR }}";
+        const result = await parseConfigurationVariable(rawValue);
+        expect(result).toEqual({ key: "MY_ENV_VAR" });
+    });
+
+    it("should parse backend functions variable correctly", async () => {
+        const rawValue = "${{ backend.functions.myFunction.url }}";
+        const result = await parseConfigurationVariable(rawValue);
+        expect(result).toEqual({ path: "backend.functions.myFunction", field: "url" });
+    });
+
+    it("should return raw value if no match is found", async () => {
+        const rawValue = "plainValue";
+        const result = await parseConfigurationVariable(rawValue);
+        expect(result).toEqual({ value: "plainValue" });
+    });
+
+    it("should handle cases with spaces correctly for env variables", async () => {
+        const rawValue = "${{ env.MY_ENV_VAR   }}";
+        const result = await parseConfigurationVariable(rawValue);
+        expect(result).toEqual({ key: "MY_ENV_VAR" });
+    });
+
+    it("should handle cases with spaces correctly for backend functions", async () => {
+        const rawValue = "${{   backend.functions.myFunction.url   }}";
+        const result = await parseConfigurationVariable(rawValue);
+        expect(result).toEqual({ path: "backend.functions.myFunction", field: "url" });
+    });
+
+    it("should return raw value if format is incorrect", async () => {
+        const rawValue = "${{ some.incorrect.format }";
+        const result = await parseConfigurationVariable(rawValue);
+        expect(result).toEqual({ value: rawValue });
+    });
+});