Ed 599 integration

.env.sample

@@ -35,9 +35,24 @@ ML_SURVEY_SERVICE_URL = "http://ml-survey-service:3000"
 # ML Project Service Service
 ML_PROJECT_SERVICE_URL = "http://ml-project-service:3000"                            // ML Project service url
 
-# Elastic search configurations 
-ELASTICSEARCH_COMMUNICATIONS_ON_OFF = "ON/OFF"                                  // Elastic search enable/disable flag E.g. ON/OFF
-ELASTICSEARCH_HOST_URL = "http://127.0.0.1:9200"                                // Elastic search host url
-ELASTICSEARCH_ENTITIES_INDEX = "entities"                                       // Elastic search index name for storing entities
-
+#USER service
 USER_SERVICE_URL = "http://user-service:3000"                                   // Base url of the sunbird enviornment
+
+CSV_REPORTS_PATH = "public/report"                                              // Report path
+
+APP_PORTAL_BASE_URL = "https://dev.sunbirded.org"
+
+FORM_SERVICE_URL = "http://player:3000"                                         // Base url for form search
+
+# Oracle Cloud Configuration                                                    
+OCI_ACCESS_KEY_ID = '23b90..............d01d'                                   // Oracle cloud storage access key Id
+OCI_SECRET_ACCESS_KEY = '22levMw5Ci............SmNE='                           // Oracle cloud storage secret access key 
+OCI_BUCKET_NAME = 'oracle cloud bucket name'                                    // Oracle cloud bucket name
+OCI_BUCKET_REGION = 'ap-hyderabad-1'                                            // Oracle cloud bucket region
+OCI_BUCKET_ENDPOINT = 'https://pmt5.compat.storage.ap-h1.oraclecloud.com'       // Oracle cloud bucket endPoint 
+
+# KAFKA Configurations
+KAFKA_COMMUNICATIONS_ON_OFF = "ON/OFF"                                               // Kafka enable or disable communication flag
+KAFKA_URL = "100.0.0.1:9092"                                                 // IP address of kafka server with port without HTTP
+KAFKA_GROUP_ID = "mlcore"                                                       // Kafka group id 
+PROGRAM_USERS_JOINED_TOPIC = "dev.programuser.info"                              // Kafka submission topic for pushing program joined user's data

.github/pull_request_template.md

@@ -0,0 +1,36 @@
+
+# Description
+These recommendations are intended to promote code quality and team communication during software development. They cover a variety of topics, including ensuring that pull requests are submitted to the correct branch, documenting new methods, preserving consistency across many services, and avoiding typical blunders like accessing APIs or DB queries within loops. Sensitive data should not be uploaded, and changes to environment variables or database models should be executed consistently. Teams may work more effectively and develop higher-quality software by adhering to these standards.
+
+
+## Type of change
+Please choose appropriate options.
+
+- [ ]  Bug fix (non-breaking change which fixes an issue)
+- [ ]  New feature (non-breaking change which adds functionality)
+- [ ]  Breaking change (fix or feature that would cause existing functionality to not work as expected)
+- [ ]  Enhancement (additive changes to improve performance)
+- [ ]  This change requires a documentation update
+
+## Checklist 
+
+- [ ]  It's critical to avoid making needless file modifications in contributions, such as adding new lines, console logs, or additional spaces, to guarantee cleaner and more efficient code. Furthermore, eliminating unnecessary imports from a file might enhance code readability and efficiency.
+- [ ]  Ensure that the pull request is assigned to the right base branch and that the development branch name contains the JIRA Task Id. Furthermore, each commit message should include the JIRA Task Id in the manner "ED-100: message".
+- [ ]  Only update packages if it is mentioned and authorized in the design document, and make sure that you have the required permissions.
+- [ ]  Avoid making API and database queries inside a loop as it can lead to performance issues and slow down the system.
+- [ ]  When calling another function inside a given function, add comments explaining the purpose and meaning of the passed arguments and expected return values.
+- [ ]  If adding a blank argument in a function, add a comment explaining the reason for the blank argument.
+- [ ]  Before submitting a pull request, do a self-review of your code to ensure there are no conflicts with the base branch and all comments have been addressed.
+- [ ]  Before merging a pull request, it's important to have other team members review it to catch any potential errors or issues
+- [ ]  To maintain code integrity, it's important to remove all related changes when removing code during a code review.
+- [ ]  If new constants, endpoints, or utility functions are introduced, it is important to check if they already exist in the service to avoid any duplication.
+- [ ]  Whenever a new environment variable is added to a service, it's important to ensure that the necessary changes are made to related files such as ".env.sample" and "envVariables.js" to maintain consistency and avoid errors. Additionally, the new environment variable should be added to the devops repository to ensure that it is properly documented and accessible to the team.
+- [ ]  When adding a new function to a service, it is important to document it with relevant information such as the name, parameters, and return value in a consistent format across all services. Additionally, if there are any changes to the API response, ensure that the documentation in the controllers is updated accordingly. 
+- [ ]  Write a clear and concise commit message that describes the changes made.
+- [ ]  Maintain consistent function signature and code across all services when adding a function to multiple services. Implement changes to database models in all services that use the same model. 
+- [ ]  Use only let and const. Do not use var.
+- [ ]  Make common functions for repetitive code blocks.
+- [ ]  Avoid uploading sensitive information such as secret tokens or passwords in pull requests to ensure data security.
+- [ ]  Maintain consistent indentation and spacing throughout the code.
+
+

.gitignore

@@ -60,6 +60,9 @@ typings/
 # Ignore all credentials gcp,fcm
 config/credentials/*
 
+# Ignore public directory
+public/*
+
 # next.js build output
 .next
 
@@ -69,3 +72,4 @@ config/credentials/*
 *.DS_Store
 
 package-lock.json
+keycloak-public-keys/

Jenkinsfile

@@ -5,41 +5,35 @@ node('build-slave') {
         String ANSI_BOLD = "\u001B[1m"
         String ANSI_RED = "\u001B[31m"
         String ANSI_YELLOW = "\u001B[33m"
+
         ansiColor('xterm') {
-            stage('Checkout') {
-                if (!env.hub_org) {
-                    println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL)
-                    error 'Please resolve the errors and rerun..'
-                }
-                else
-                    println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
-                cleanWs()
-                if (params.github_release_tag == "") {
-                    checkout scm
-                    commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
-                    branch_name = sh(script: 'git name-rev --name-only HEAD | rev | cut -d "/" -f1| rev', returnStdout: true).trim()
-                    build_tag = branch_name + "_" + commit_hash
-                    println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag not specified, using the latest commit hash: " + commit_hash + ANSI_NORMAL)
-                }
-                else {
-                    def scmVars = checkout scm
-                    checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$params.github_release_tag"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
-                    build_tag = params.github_release_tag
-                    println(ANSI_BOLD + ANSI_YELLOW + "github_release_tag specified, building from github_release_tag: " + params.github_release_tag + ANSI_NORMAL)
+            timestamps {
+                stage('Checkout') {
+                    if (!env.hub_org) {
+                        println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL)
+                        error 'Please resolve the errors and rerun..'
+                    } else
+                        println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
                 }
+                // cleanWs()
+                checkout scm
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                build_tag = sh(script: "echo " + params.github_release_tag.split('/')[-1] + "_" + commit_hash + "_" + env.BUILD_NUMBER, returnStdout: true).trim()
                 echo "build_tag: " + build_tag
-            }
-            stage('Build') {
-                env.NODE_ENV = "build"
-                print "Environment will be : ${env.NODE_ENV}"
-                sh('git submodule update --init')
-                sh('git submodule update --init --recursive --remote')
-                sh('chmod 777 build.sh')
-                sh("./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}")
-            }
-            stage('ArchiveArtifacts') {
+
+                stage('Build') {
+                   env.NODE_ENV = "build"
+                   print "Environment will be : ${env.NODE_ENV}"
+                   sh('git submodule update --init')
+                   sh('git submodule update --init --recursive --remote')
+                   sh('chmod 777 build.sh')
+                   sh("./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}")
+                  }
+
+                stage('ArchiveArtifacts') {
                 archiveArtifacts "metadata.json"
                 currentBuild.description = "${build_tag}"
+               }
             }
         }
     }

README.md

@@ -1,3 +1,5 @@
 # ml-core-service
-Centralised Service to support other Services
+Centralised Service to support other Services.
+It is used by ml-survey, ml-project
+
 

app.js

@@ -45,18 +45,13 @@ app.use(express.static("public"));
 
 
 app.all("*", (req, res, next) => {
-  console.log("-------Request log starts here------------------");
-  console.log(
-    "%s %s on %s from ",
-    req.method,
-    req.url,
-    new Date(),
-    req.headers["user-agent"]
-  );
-  console.log("Request Headers: ", req.headers);
-  console.log("Request Body: ", req.body);
-  console.log("Request Files: ", req.files);
-  console.log("-------Request log ends here------------------");
+  console.log({"Debugging ML Core Service": true});
+  console.log("<------------Request log starts here------------------>");
+  console.log("Request URL: ", req.url);
+  console.log("Request Headers: ", JSON.stringify(req.headers));
+  console.log("Request Body: ", JSON.stringify(req.body));
+  // console.log("Request Files: ", req.files);
+  console.log("<--------------Request log ends here------------------>");
   next();
 });
 

auto_build_deploy

@@ -0,0 +1,55 @@
+@Library('deploy-conf') _
+node('build-slave') {
+    try {
+        String ANSI_GREEN = "\u001B[32m"
+        String ANSI_NORMAL = "\u001B[0m"
+        String ANSI_BOLD = "\u001B[1m"
+        String ANSI_RED = "\u001B[31m"
+        String ANSI_YELLOW = "\u001B[33m"
+
+        ansiColor('xterm') {
+            timestamps {
+                stage('Checkout') {
+                    tag_name = env.JOB_NAME.split("/")[-1]
+                    pre_checks()
+                    if (!env.hub_org) {
+                        println(ANSI_BOLD + ANSI_RED + "Uh Oh! Please set a Jenkins environment variable named hub_org with value as registery/sunbidrded" + ANSI_NORMAL)
+                        error 'Please resolve the errors and rerun..'
+                    } else
+                        println(ANSI_BOLD + ANSI_GREEN + "Found environment variable named hub_org with value as: " + hub_org + ANSI_NORMAL)
+                }
+                cleanWs()
+                def scmVars = checkout scm
+                checkout scm: [$class: 'GitSCM', branches: [[name: "refs/tags/$tag_name"]], userRemoteConfigs: [[url: scmVars.GIT_URL]]]
+                build_tag = tag_name + "_" + env.BUILD_NUMBER
+                commit_hash = sh(script: 'git rev-parse --short HEAD', returnStdout: true).trim()
+                artifact_version = tag_name + "_" + commit_hash
+                echo "build_tag: " + build_tag
+
+              //  stage Build
+                    env.NODE_ENV = "build"
+                    print "Environment will be : ${env.NODE_ENV}"
+                    sh('git submodule update --init')
+                    sh('git submodule update --init --recursive --remote')
+                    sh('chmod 777 build.sh')
+                    sh("./build.sh ${build_tag} ${env.NODE_NAME} ${hub_org}")
+
+
+             //   stage ArchiveArtifacts
+                    archiveArtifacts "metadata.json"
+                    currentBuild.description = "${build_tag}"
+
+            }
+        }
+        currentBuild.result = "SUCCESS"    
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()  
+        auto_build_deploy()
+    }
+    catch (err) {
+        currentBuild.result = "FAILURE"
+        slack_notify(currentBuild.result, tag_name)
+        email_notify()
+        throw err
+    }
+}

config/db/elastic-search.js

@@ -7,37 +7,37 @@
 
 
 //dependencies
-const { Client : esClient } = require('@elastic/elasticsearch');
-
-/**
- * Elastic search connection.
- * @function
- * @name connect
- * @return {Object} elastic search client 
- */
-
-var connect = function () {
-
-  const elasticSearchClient = new esClient({
-    node: process.env.ELASTICSEARCH_HOST_URL,
-    maxRetries: 5,
-    requestTimeout: 60000,
-    sniffOnStart: process.env.ELASTIC_SEARCH_SNIFF_ON_START
-  });
-
-  elasticSearchClient.ping({
-  }, function (error) {
-    if (error) {
-      console.log(error);
-    } else {
-      console.log('Elasticsearch connection established.');
-    }
-  });
-
-  return {
-    client: elasticSearchClient
-  };
-
-};
-
-module.exports = connect;
+// const { Client : esClient } = require('@elastic/elasticsearch');
+
+// /**
+//  * Elastic search connection.
+//  * @function
+//  * @name connect
+//  * @return {Object} elastic search client 
+//  */
+
+// var connect = function () {
+
+//   const elasticSearchClient = new esClient({
+//     node: process.env.ELASTICSEARCH_HOST_URL,
+//     maxRetries: 5,
+//     requestTimeout: 60000,
+//     sniffOnStart: process.env.ELASTIC_SEARCH_SNIFF_ON_START
+//   });
+
+//   elasticSearchClient.ping({
+//   }, function (error) {
+//     if (error) {
+//       console.log(error);
+//     } else {
+//       console.log('Elasticsearch connection established.');
+//     }
+//   });
+
+//   return {
+//     client: elasticSearchClient
+//   };
+
+// };
+
+// module.exports = connect;

config/db/mongodb.js

@@ -70,11 +70,16 @@ var databaseConfiguration = function () {
     return model;
   };
 
+  var getCollection = function (modelName) {
+    return db.collection(modelName);
+  } 
+
   return {
     database: db,
     createModel: createModel,
     ObjectId: objectId,
-    models: db.models
+    models: db.models,
+    getCollection : getCollection
   };
 };
 

config/globals.js

@@ -99,4 +99,7 @@ module.exports = function () {
     }
   });
 
+  //define cache as global variable
+  global.cache = require(ROOT_PATH+"/generics/helpers/cache");
+
 };

config/index.js

@@ -23,8 +23,18 @@ let db_connect = function () {
   * @name elasticsearch_connect
 */
 
-let elasticsearch_connect = function () {
-  global.elasticsearch = require("./db/elastic-search")();
+// let elasticsearch_connect = function () {
+//   global.elasticsearch = require("./db/elastic-search")();
+// };
+
+/**
+  * Kafka connection.
+  * @function
+  * @name kafka_connect
+*/
+
+const kafka_connect = function() {
+  global.kafkaClient = require("./kafka")();
 };
 
 const configuration = {
@@ -35,6 +45,7 @@ const configuration = {
 };
 
 db_connect();
-elasticsearch_connect();
+kafka_connect();
+// elasticsearch_connect();
 
 module.exports = configuration;