Add ProfileProvider class to support read profile config from PROFILE_DB.

.azure-pipelines/build-template.yml

@@ -134,6 +134,7 @@ jobs:
         sudo sed -ri 's/^# unixsocket/unixsocket/' /etc/redis/redis.conf
         sudo sed -ri 's/^unixsocketperm .../unixsocketperm 777/' /etc/redis/redis.conf
         sudo sed -ri 's/redis-server.sock/redis.sock/' /etc/redis/redis.conf
+        sudo sed -ri 's/^databases .*/databases 17/' /etc/redis/redis.conf
         sudo service redis-server restart
         sudo mkdir /usr/local/yang-models
 

common/Makefile.am

@@ -63,6 +63,7 @@ libswsscommon_la_SOURCES = \
     linkcache.cpp             \
     portmap.cpp               \
     pubsub.cpp                \
+    profileprovider.cpp       \
     tokenize.cpp              \
     exec.cpp                  \
     saiaclschema.cpp          \

common/configdb.cpp

@@ -244,24 +244,7 @@ void ConfigDBConnector_Native::mod_config(const map<string, map<string, map<stri
 map<string, map<string, map<string, string>>> ConfigDBConnector_Native::get_config()
 {
     auto& client = get_redis_client(m_db_name);
-    auto const& keys = client.keys("*");
-    map<string, map<string, map<string, string>>> data;
-    for (string key: keys)
-    {
-        size_t pos = key.find(m_table_name_separator);
-        if (pos == string::npos) {
-            continue;
-        }
-        string table_name = key.substr(0, pos);
-        string row = key.substr(pos + 1);
-        auto const& entry = client.hgetall<map<string, string>>(key);
-
-        if (!entry.empty())
-        {
-            data[table_name][row] = entry;
-        }
-    }
-    return data;
+    return client.getall();
 }
 
 std::string ConfigDBConnector_Native::getKeySeparator() const

common/database_config.json

@@ -91,6 +91,11 @@
             "id" : 14,
             "separator": ":",
             "instance" : "redis"
+        },
+        "PROFILE_DB" : {
+            "id" : 15,
+            "separator": ":",
+            "instance" : "redis"
         }
     },
     "VERSION" : "1.0"

common/dbconnector.cpp

@@ -191,7 +191,7 @@ void SonicDBConfig::validateNamespace(const string &netns)
 
     SWSS_LOG_ENTER();
 
-    // With valid namespace input and database_global.json is not loaded, ask user to initializeGlobalConfig first
+    // With valid namespace input and database_global.json is not loaded, ask user to initializeGlobalConfig first.
     if(!netns.empty())
     {
         // If global initialization is not done, ask user to initialize global DB Config first.
@@ -905,3 +905,26 @@ void DBConnector::del(const std::vector<std::string>& keys)
 
     pipe.flush();
 }
+
+map<string, map<string, map<string, string>>> DBConnector::getall()
+{
+    const string separator = SonicDBConfig::getSeparator(this);
+    auto const& keys = this->keys("*");
+    map<string, map<string, map<string, string>>> data;
+    for (string key: keys)
+    {
+        size_t pos = key.find(separator);
+        if (pos == string::npos) {
+            continue;
+        }
+        string table_name = key.substr(0, pos);
+        string row = key.substr(pos + 1);
+        auto const& entry = this->hgetall<map<string, string>>(key);
+
+        if (!entry.empty())
+        {
+            data[table_name][row] = entry;
+        }
+    }
+    return data;
+}

common/dbconnector.h

@@ -248,6 +248,7 @@ class DBConnector : public RedisContext
 
     bool flushdb();
 
+    std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> getall();
 private:
     void setNamespace(const std::string &netns);
 

common/profileprovider.cpp

@@ -0,0 +1,214 @@
+#include <boost/algorithm/string.hpp>
+#include <cassert>
+#include <iostream>
+#include <map>
+#include <string>
+#include <tuple>
+#include <vector>
+#include <dirent.h> 
+#include <stdio.h> 
+
+#include "profileprovider.h"
+#include "logger.h"
+#include "table.h"
+#include "schema.h"
+
+using namespace std;
+using namespace swss;
+
+ProfileProvider& ProfileProvider::instance()
+{
+    static ProfileProvider instance;
+    return instance;
+}
+
+shared_ptr<string> ProfileProvider::getConfig(const string &table, const string &key, const string &field, DBConnector* cfgDbConnector)
+{
+    assert(!table.empty());
+    assert(!key.empty());
+    assert(!field.empty());
+
+    auto staticConfig = getConfigs(table, key, cfgDbConnector);
+    auto result = staticConfig.find(field);
+    if (result != staticConfig.end())
+    {
+        return make_shared<string>(result->second);
+    }
+
+    // Config not found is different with 'empty' config
+    return nullptr;
+}
+
+bool ProfileProvider::appendConfigs(const string &table, const string &key, vector<pair<string, string> > &values, DBConnector* cfgDbConnector)
+{
+    assert(!table.empty());
+    assert(!key.empty());
+
+    SWSS_LOG_DEBUG("DefaultValueProvider::AppendDefaultValues %s %s\n", table.c_str(), key.c_str());
+
+    auto staticConfig = getConfigs(table, key, cfgDbConnector);
+
+    map<string, string> existedValues;
+    for (auto& fieldValuePair : values)
+    {
+        existedValues.emplace(fieldValuePair.first, fieldValuePair.second);
+    }
+
+    bool appendValues = false;
+    for (auto& fieldValuePair : staticConfig)
+    {
+        auto findresult = existedValues.find(fieldValuePair.first);
+        if (findresult == existedValues.end())
+        {
+            appendValues = true;
+            values.emplace_back(fieldValuePair.first, fieldValuePair.second);
+        }
+    }
+
+    return appendValues;
+}
+
+map<string, string> ProfileProvider::getConfigs(const string &table, const string &key, DBConnector* cfgDbConnector)
+{
+    if (itemDeleted(table, key, cfgDbConnector))
+    {
+        SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs item %s %s deleted.\n", table.c_str(), key.c_str());
+        map<string, string> map;
+        return map;
+    }
+
+    auto& staticCfgDbConnector = getStaticCfgDBConnector(cfgDbConnector);
+    auto itemkey = getKeyName(table, key, &staticCfgDbConnector);
+    return staticCfgDbConnector.hgetall<map<string, string>>(itemkey);
+}
+
+map<string, map<string, map<string, string>>> ProfileProvider::getConfigs(DBConnector* cfgDbConnector)
+{
+    auto& staticCfgDbConnector = getStaticCfgDBConnector(cfgDbConnector);
+    auto configs = staticCfgDbConnector.getall();
+
+    // If a profile item mark as 'deleted', it's shoud not exist in result.
+    list<pair<string, string>> deletedItems;
+    for(auto const& tableItem: configs)
+    {
+        auto table = tableItem.first;
+        for(auto const& item: tableItem.second)
+        {
+            auto key = item.first;
+            if (itemDeleted(table, key, cfgDbConnector))
+            {
+                SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs item %s %s deleted.\n", table.c_str(), key.c_str());
+                deletedItems.push_back(make_pair(table, key));
+            }
+        }
+    }
+
+    for(auto const& deletedItem: deletedItems)
+    {
+        auto table = deletedItem.first;
+        auto key = deletedItem.second;
+        SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs remove deleted item %s %s from result.\n", table.c_str(), key.c_str());
+        configs[table].erase(key);
+    }
+
+    return configs;
+}
+
+vector<string> ProfileProvider::getKeys(const string &table, DBConnector* cfgDbConnector)
+{
+    auto& staticCfgDbConnector = getStaticCfgDBConnector(cfgDbConnector);
+    auto pattern = getKeyName(table, "*", &staticCfgDbConnector);
+    auto keys = staticCfgDbConnector.keys(pattern);
+
+    const auto separator = SonicDBConfig::getSeparator(&staticCfgDbConnector);
+    vector<string> result;
+    for(auto const& itemKey: keys)
+    {
+        size_t pos = itemKey.find(separator);
+        if (pos == string::npos)
+        {
+            SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs can't find separator %s in %s.\n", separator.c_str(), itemKey.c_str());
+            continue;
+        }
+
+        auto row = itemKey.substr(pos + 1);
+        if (!itemDeleted(table, row, cfgDbConnector))
+        {
+            result.push_back(row);
+        }
+        else
+        {
+            SWSS_LOG_DEBUG("DefaultValueProvider::GetConfigs item %s %s deleted.\n", table.c_str(), row.c_str());
+        }
+    }
+
+    return result;
+}
+
+ProfileProvider::ProfileProvider()
+{
+}
+
+ProfileProvider::~ProfileProvider()
+{
+}
+
+bool ProfileProvider::tryRevertItem(const string &table, const string &key, DBConnector* cfgDbConnector)
+{
+    if (itemDeleted(table, key, cfgDbConnector))
+    {
+        revertItem(table, key, cfgDbConnector);
+        return true;
+    }
+
+    return false;
+}
+
+bool ProfileProvider::tryDeleteItem(const string &table, const string &key, DBConnector* cfgDbConnector)
+{
+    if (!itemDeleted(table, key, cfgDbConnector))
+    {
+        deleteItem(table, key, cfgDbConnector);
+        return true;
+    }
+
+    return false;
+}
+
+bool ProfileProvider::itemDeleted(const string &table, const string &key, DBConnector* cfgDbConnector)
+{
+    auto deletedkey = getDeletedKeyName(table, key, cfgDbConnector);
+    return cfgDbConnector->exists(deletedkey) == true;
+}
+
+void ProfileProvider::deleteItem(const string &table, const string &key, DBConnector* cfgDbConnector)
+{
+    auto deletedkey = getDeletedKeyName(table, key, cfgDbConnector);
+    // Only need deletedkey to mark the item is deleted.
+    cfgDbConnector->hset(deletedkey, "", "");
+}
+
+void ProfileProvider::revertItem(const string &table, const string &key, DBConnector* cfgDbConnector)
+{
+    auto deletedkey = getDeletedKeyName(table, key,cfgDbConnector);
+    cfgDbConnector->del(deletedkey);
+}
+
+DBConnector& ProfileProvider::getStaticCfgDBConnector(DBConnector* cfgDbConnector)
+{
+    auto ns = cfgDbConnector->getNamespace();
+    auto result = m_staticCfgDBMap.find(ns);
+    if (result != m_staticCfgDBMap.end())
+    {
+        return result->second;
+    }
+
+    // Create new DBConnector instance to PROFILE_DB
+    const string staticDbName = "PROFILE_DB";
+    m_staticCfgDBMap.emplace(piecewise_construct,
+              forward_as_tuple(ns),
+              forward_as_tuple(staticDbName, SonicDBConfig::getDbId(staticDbName, ns), true, ns));
+
+    result = m_staticCfgDBMap.find(ns);
+    return result->second;
+}

common/profileprovider.h

@@ -0,0 +1,61 @@
+#pragma once
+
+#include <map>
+#include <string>
+#include <vector>
+#include "common/table.h"
+#include "common/dbconnector.h"
+#include "common/converter.h"
+
+namespace swss {
+
+const std::string DELETED_KEY_SEPARATOR = "_";
+
+class ProfileProvider
+{
+public:
+    static ProfileProvider& instance();
+
+    bool appendConfigs(const std::string &table, const std::string &key, std::vector<std::pair<std::string, std::string> > &values, DBConnector* cfgDbConnector);
+
+    std::shared_ptr<std::string> getConfig(const std::string &table, const std::string &key, const std::string &field, DBConnector* cfgDbConnector);
+
+    std::map<std::string, std::string> getConfigs(const std::string &table, const std::string &key, DBConnector* cfgDbConnector);
+
+    std::map<std::string, std::map<std::string, std::map<std::string, std::string>>> getConfigs(DBConnector* cfgDbConnector);
+
+    std::vector<std::string> getKeys(const std::string &table, DBConnector* cfgDbConnector);
+
+    bool tryRevertItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector);
+
+    bool tryDeleteItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector);
+
+    std::string getDeletedKeyName(const std::string &table, const std::string &key, DBConnector* dbConnector)
+    {
+        auto itemKey = to_upper(table) + DELETED_KEY_SEPARATOR + key;
+        return getKeyName(PROFILE_DELETE_TABLE, itemKey, dbConnector);
+    }
+
+private:
+    ProfileProvider();
+    ~ProfileProvider();
+
+    std::string getKeyName(const std::string &table, const std::string &key, DBConnector* dbConnector)
+    {
+        const auto separator = SonicDBConfig::getSeparator(dbConnector);
+        // Profile DB follow Config DB: table name is case insensetive.
+        return to_upper(table) + separator + key;
+    }
+
+    bool itemDeleted(const std::string &table, const std::string &key, DBConnector* cfgDbConnector);
+
+    void deleteItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector);
+
+    void revertItem(const std::string &table, const std::string &key, DBConnector* cfgDbConnector);
+
+    DBConnector& getStaticCfgDBConnector(DBConnector* cfgDbConnector);
+
+    std::map<std::string, DBConnector> m_staticCfgDBMap;
+};
+
+}

common/schema.h

@@ -490,6 +490,10 @@ namespace swss {
 
 #define STATE_FLOW_COUNTER_CAPABILITY_TABLE_NAME    "FLOW_COUNTER_CAPABILITY_TABLE"
 
+/***** PROFILE DATABASE *****/
+
+#define PROFILE_DELETE_TABLE                  "PROFILE_DELETE"
+
 /***** MISC *****/
 
 #define IPV4_NAME "IPv4"