fix: better support for custom gitlab ports

README.md

@@ -42,6 +42,7 @@ Get rid of all those dev specific shell scripts and make files.
     * [Decorators](#decorators)
     * [Includes](#includes)
     * [Artifacts](#artifacts)
+    * [Self Hosted Custom Ports](#self-hosted-custom-ports)
 * [Development](#development)
     * [Scripts](#scripts)
     * [Package binaries](#package-binaries)
@@ -358,6 +359,18 @@ handling for shell jobs.
 
 Docker executor copies artifacts to and from .gitlab-ci-local/artifacts
 
+### Self Hosted Custom Ports
+
+If your self-host gitlab instance uses custom ports, it is recommended to manually define the `CI_SERVER_PORT` and `CI_SERVER_SHELL_SSH_PORT`
+
+```yaml
+---
+# $CWD/.gitlab-ci-local-variables.yml
+
+CI_SERVER_PORT: 8443
+CI_SERVER_SHELL_SSH_PORT: 8022
+```
+
 ## Development
 
 You need nodejs 18+

src/job.ts

@@ -74,6 +74,9 @@ export type JobRule = {
 };
 
 export class Job {
+    private generateJobId (): number {
+        return Math.floor(Math.random() * 1000000);
+    }
 
     static readonly illegalJobNames = new Set([
         "include", "local_configuration", "image", "services",
@@ -136,7 +139,7 @@ export class Job {
         this.gitData = opt.gitData;
         this.name = opt.name;
         this.baseName = opt.baseName;
-        this.jobId = Math.floor(Math.random() * 1000000);
+        this.jobId = this.generateJobId();
         this.jobData = opt.data;
         this.pipelineIid = opt.pipelineIid;
 
@@ -170,8 +173,8 @@ export class Job {
         predefinedVariables["CI_JOB_NAME_SLUG"] = `${this.name.replace(/[^a-z\d]+/ig, "-").replace(/^-/, "").slice(0, 63).replace(/-$/, "").toLowerCase()}`;
         predefinedVariables["CI_JOB_STAGE"] = `${this.stage}`;
         predefinedVariables["CI_PROJECT_DIR"] = ciProjectDir;
-        predefinedVariables["CI_JOB_URL"] = `https://${gitData.remote.host}/${gitData.remote.group}/${gitData.remote.project}/-/jobs/${this.jobId}`; // Changes on rerun.
-        predefinedVariables["CI_PIPELINE_URL"] = `https://${gitData.remote.host}/${gitData.remote.group}/${gitData.remote.project}/pipelines/${this.pipelineIid}`;
+        predefinedVariables["CI_JOB_URL"] = `${this._variables["CI_SERVER_URL"]}/${gitData.remote.group}/${gitData.remote.project}/-/jobs/${this.jobId}`; // Changes on rerun.
+        predefinedVariables["CI_PIPELINE_URL"] = `${this._variables["CI_SERVER_URL"]}/${gitData.remote.group}/${gitData.remote.project}/pipelines/${this.pipelineIid}`;
         predefinedVariables["CI_ENVIRONMENT_NAME"] = this.environment?.name ?? "";
         predefinedVariables["CI_ENVIRONMENT_SLUG"] = this.environment?.name?.replace(/[^a-z\d]+/ig, "-").replace(/^-/, "").slice(0, 23).replace(/-$/, "").toLowerCase() ?? "";
         predefinedVariables["CI_ENVIRONMENT_URL"] = this.environment?.url ?? "";

src/parser-includes.ts

@@ -105,12 +105,13 @@ export class ParserIncludes {
                     includeDatas = includeDatas.concat(await this.init(fileDoc, opts));
                 }
             } else if (value["component"]) {
-                const {domain, projectPath, componentName, ref} = this.parseIncludeComponent(value["component"]);
+                const {domain, port, projectPath, componentName, ref} = this.parseIncludeComponent(value["component"]);
                 // converts component to project
                 const files = [`${componentName}.yml`, `${componentName}/template.yml`, null];
 
                 for (const f of files) {
-                    assert(f !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the file [${files}] must exists in \`${domain}/${projectPath}\``);
+                    assert(f !== null, `This GitLab CI configuration is invalid: component: \`${value["component"]}\`. One of the files [${files}] must exist in \`${domain}` +
+                                        (port ? `:${port}` : "") + `/${projectPath}\``);
 
                     const isLocalComponent = projectPath === `${gitData.remote.group}/${gitData.remote.project}` && ref === gitData.commit.SHA;
                     if (isLocalComponent) {
@@ -199,15 +200,16 @@ export class ParserIncludes {
         };
     }
 
-    static parseIncludeComponent (component: string): {domain: string; projectPath: string; componentName: string; ref: string} {
+    static parseIncludeComponent (component: string): {domain: string; port: string; projectPath: string; componentName: string; ref: string} {
         assert(!component.includes("://"), `This GitLab CI configuration is invalid: component: \`${component}\` should not contain protocol`);
         // eslint-disable-next-line no-useless-escape
-        const pattern = /(?<domain>[^/\s]+)\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // regexr.com/7v7hm
+        const pattern = /(?<domain>[^/:\s]+)(:(?<port>[0-9]+))?\/(?<projectPath>.+)\/(?<componentName>[^@]+)@(?<ref>.+)/; // https://regexr.com/86q5d
         const gitRemoteMatch = pattern.exec(component);
 
         if (gitRemoteMatch?.groups == null) throw new Error(`This is a bug, please create a github issue if this is something you're expecting to work. input: ${component}`);
         return {
             domain: gitRemoteMatch.groups["domain"],
+            port: gitRemoteMatch.groups["port"],
             projectPath: gitRemoteMatch.groups["projectPath"],
             componentName: `templates/${gitRemoteMatch.groups["componentName"]}`,
             ref: gitRemoteMatch.groups["ref"],
@@ -239,7 +241,7 @@ export class ParserIncludes {
                 await Utils.bash(`
                     cd ${cwd}/${stateDir} \\
                         && git clone --branch "${ref}" -n --depth=1 --filter=tree:0 \\
-                                ${remote.schema}://${remote.host}/${project}.git \\
+                                ${remote.schema}://${remote.host}:${remote.port}/${project}.git \\
                                 ${cwd}/${target}.${ext} \\
                         && cd ${cwd}/${target}.${ext} \\
                         && git sparse-checkout set --no-cone ${normalizedFile} \\

src/parser.ts

@@ -98,8 +98,8 @@ export class Parser {
         const fetchIncludes = argv.fetchIncludes;
         const gitData = await GitData.init(cwd, writeStreams);
         const variablesFromFiles = await VariablesFromFiles.init(argv, writeStreams, gitData);
-        const predefinedVariables = initPredefinedVariables({gitData, argv});
         const envMatchedVariables = Utils.findEnvMatchedVariables(variablesFromFiles);
+        const predefinedVariables = initPredefinedVariables({gitData, argv, envMatchedVariables});
         const variables = {...predefinedVariables, ...envMatchedVariables, ...argv.variable};
         const expanded = Utils.expandVariables(variables);
 

src/predefined-variables.ts

@@ -3,10 +3,45 @@ import {GitData} from "./git-data.js";
 
 type PredefinedVariablesOpts = {
     gitData: GitData;
-    argv: {unsetVariables: string[]};
+    argv: {
+        unsetVariables: string[];
+        variable: {[key: string]: string};
+    };
+    envMatchedVariables: {[key: string]: string};
 };
 
-export function init ({gitData, argv}: PredefinedVariablesOpts): {[name: string]: string} {
+export function init ({gitData, argv, envMatchedVariables}: PredefinedVariablesOpts): {[name: string]: string} {
+
+    const _variables = {...envMatchedVariables, ...argv.variable};
+
+    // precedence:
+    // 1. cli option
+    // 2. gitlab variables files
+    // 3. values derieved implicitly from `git remote -v`
+    // 4. default value
+    const CI_SERVER_PROTOCOL = _variables["CI_SERVER_PROTOCOL"]
+      ?? ((gitData.remote.schema === "http" || gitData.remote.schema === "https")
+          ? gitData.remote.schema
+          : "https");
+    const CI_SERVER_PORT = _variables["CI_SERVER_PORT"]
+      ?? ((gitData.remote.schema === "http" || gitData.remote.schema === "https")
+          ? gitData.remote.port
+          : "443");
+    const CI_SERVER_SHELL_SSH_PORT = _variables["CI_SERVER_SHELL_SSH_PORT"]
+      ?? ((gitData.remote.schema === "ssh")
+          ? gitData.remote.port
+          : "22");
+    const CI_SERVER_HOST = _variables["CI_SERVER_HOST"]
+      ?? `${gitData.remote.host}`;
+    const CI_SERVER_FQDN = _variables["CI_SERVER_FQDN"]
+      ?? (CI_SERVER_PORT == "443"
+          ? gitData.remote.host
+          : `${gitData.remote.host}:${CI_SERVER_PORT}`);
+    const CI_SERVER_URL = _variables["CI_SERVER_URL"]
+      ?? `${CI_SERVER_PROTOCOL}://${CI_SERVER_FQDN}`;
+    const CI_PROJECT_ROOT_NAMESPACE = gitData.remote.group.split("/")[0];
+    const CI_PROJECT_NAMESPACE = gitData.remote.group;
+
     const predefinedVariables: {[key: string]: string} = {
         CI: "true",
         GITLAB_USER_LOGIN: gitData.user["GITLAB_USER_LOGIN"],
@@ -19,7 +54,8 @@ export function init ({gitData, argv}: PredefinedVariablesOpts): {[name: string]
         CI_PROJECT_TITLE: `${camelCase(gitData.remote.project)}`,
         CI_PROJECT_PATH: `${gitData.remote.group}/${gitData.remote.project}`,
         CI_PROJECT_PATH_SLUG: `${gitData.remote.group.replace(/\//g, "-")}-${gitData.remote.project}`.toLowerCase(),
-        CI_PROJECT_NAMESPACE: `${gitData.remote.group}`,
+        CI_PROJECT_ROOT_NAMESPACE: CI_PROJECT_ROOT_NAMESPACE,
+        CI_PROJECT_NAMESPACE: CI_PROJECT_NAMESPACE,
         CI_PROJECT_VISIBILITY: "internal",
         CI_PROJECT_ID: "1217",
         CI_COMMIT_REF_PROTECTED: "false",
@@ -28,19 +64,19 @@ export function init ({gitData, argv}: PredefinedVariablesOpts): {[name: string]
         CI_COMMIT_REF_SLUG: gitData.commit.REF_NAME.replace(/[^a-z\d]+/ig, "-").replace(/^-/, "").slice(0, 63).replace(/-$/, "").toLowerCase(),
         CI_COMMIT_TIMESTAMP: gitData.commit.TIMESTAMP,
         CI_PIPELINE_CREATED_AT: new Date().toISOString().split(".")[0] + "Z",
-        CI_JOB_STARTED_AT: new Date().toISOString().split(".")[0] + "Z",
         CI_COMMIT_TITLE: "Commit Title", // First line of commit message.
         CI_COMMIT_MESSAGE: "Commit Title\nMore commit text", // Full commit message
         CI_COMMIT_DESCRIPTION: "More commit text",
         CI_DEFAULT_BRANCH: gitData.branches.default,
         CI_PIPELINE_SOURCE: "push",
-        CI_SERVER_FQDN: `${gitData.remote.host}`,
-        CI_SERVER_HOST: `${gitData.remote.host}`,
-        CI_SERVER_PORT: `${gitData.remote.port}`,
-        CI_SERVER_URL: `https://${gitData.remote.host}:443`,
-        CI_SERVER_PROTOCOL: "https",
-        CI_API_V4_URL: `https://${gitData.remote.host}/api/v4`,
-        CI_PROJECT_URL: `https://${gitData.remote.host}/${gitData.remote.group}/${gitData.remote.project}`,
+        CI_SERVER_FQDN: CI_SERVER_FQDN,
+        CI_SERVER_HOST: CI_SERVER_HOST,
+        CI_SERVER_PORT: CI_SERVER_PORT,
+        CI_SERVER_SHELL_SSH_PORT: CI_SERVER_SHELL_SSH_PORT,
+        CI_SERVER_URL: CI_SERVER_URL,
+        CI_SERVER_PROTOCOL: CI_SERVER_PROTOCOL,
+        CI_API_V4_URL: `${CI_SERVER_URL}/api/v4`,
+        CI_PROJECT_URL: `${CI_SERVER_URL}/${gitData.remote.group}/${gitData.remote.project}`,
         CI_TEMPLATE_REGISTRY_HOST: "registry.gitlab.com",
         GITLAB_CI: "false",
     };

src/utils.ts

@@ -345,7 +345,7 @@ ${evalStr}
             case "http":
             case "https": {
                 try {
-                    const {status} = await axios.get(`${protocol}://${domain}/${projectPath}/-/raw/${ref}/${file}`);
+                    const {status} = await axios.get(`${protocol}://${domain}:${port}/${projectPath}/-/raw/${ref}/${file}`);
                     return (status === 200);
                 } catch (e) {
                     return false;

tests/test-cases/include-component/integration.include-component.test.ts

@@ -20,7 +20,7 @@ test.concurrent("include-component no component template file (protocol: https)"
         expect(true).toBe(false);
     } catch (e: any) {
         assert(e instanceof AssertionError, `Unexpected error thrown:\n ${e}`);
-        expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the file [templates/potato.yml,templates/potato/template.yml,] must exists in `gitlab.com/components/go`");
+        expect(e.message).toBe("This GitLab CI configuration is invalid: component: `gitlab.com/components/go/potato@0.3.1`. One of the files [templates/potato.yml,templates/potato/template.yml,] must exist in `gitlab.com/components/go`");
     }
 });
 

tests/test-cases/predefined-variables/.gitlab-ci.yml

@@ -1,16 +1,9 @@
 ---
 test-job:
+  image: busybox
   script:
-    - echo ${CI_PROJECT_NAME}
-    - echo ${CI_PROJECT_PATH}
-    - echo ${CI_PROJECT_PATH_SLUG}
-    - echo ${CI_PROJECT_NAMESPACE}
-    - echo ${CI_PROJECT_DIR}
-    - echo ${CI_DEFAULT_BRANCH}
-    - echo ${CI_REGISTRY_IMAGE}
-    - echo ${CI_NODE_INDEX}/${CI_NODE_TOTAL}
+    - env | sort | grep -Ev "^PATH|^HOSTNAME|^HOME=|^More commit text$|^SHLVL"
 
-test-job-commit-short-length:
+shell-isolation:
   script:
-    - echo ${CI_COMMIT_SHORT_SHA}
-    - "[ ${#CI_COMMIT_SHORT_SHA} -eq 8 ] || exit 1"
+    - echo ${CI_PROJECT_DIR}