Merge pull request #597 from singnet/lighthouse

feat: filecoin integration

blockchain/orginzationMetadata.go

@@ -98,17 +98,16 @@ type PaymentChannelStorageClient struct {
 	Endpoints         []string `json:"endpoints"`
 }
 
-// Construct the Organization metadata from the JSON Passed
-func InitOrganizationMetaDataFromJson(jsonData string) (metaData *OrganizationMetaData, err error) {
+// InitOrganizationMetaDataFromJson Construct the Organization metadata from the JSON Passed
+func InitOrganizationMetaDataFromJson(jsonData []byte) (metaData *OrganizationMetaData, err error) {
 	metaData = new(OrganizationMetaData)
-	err = json.Unmarshal([]byte(jsonData), &metaData)
+	err = json.Unmarshal(jsonData, &metaData)
 	if err != nil {
 		zap.L().Error("Error in unmarshalling metadata json", zap.Error(err), zap.Any("jsondata", jsonData))
 		return nil, err
 	}
 
 	// Check for mandatory validations
-
 	if err = setDerivedAttributes(metaData); err != nil {
 		zap.L().Error("Error in setting derived attributes", zap.Error(err))
 		return nil, err
@@ -166,7 +165,7 @@ func GetOrganizationMetaData() *OrganizationMetaData {
 	var err error
 	if config.GetBool(config.BlockchainEnabledKey) {
 		ipfsHash := string(getMetaDataURI())
-		metadata, err = GetOrganizationMetaDataFromIPFS(FormatHash(ipfsHash))
+		metadata, err = GetOrganizationMetaDataFromIPFS(ipfsHash)
 	} else {
 		metadata = &OrganizationMetaData{daemonGroup: &Group{}}
 	}
@@ -177,7 +176,10 @@ func GetOrganizationMetaData() *OrganizationMetaData {
 }
 
 func GetOrganizationMetaDataFromIPFS(hash string) (*OrganizationMetaData, error) {
-	jsondata := ipfsutils.GetIpfsFile(hash)
+	jsondata, err := ipfsutils.ReadFile(hash)
+	if err != nil {
+		return nil, err
+	}
 	return InitOrganizationMetaDataFromJson(jsondata)
 }
 

blockchain/serviceMetadata.go

@@ -143,18 +143,18 @@ import (
 	}
 */
 const (
-	IpfsPrefix   = "ipfs://"
 	serviceProto = "service.proto"
 )
 
 type ServiceMetadata struct {
-	Version       int                 `json:"version"`
-	DisplayName   string              `json:"display_name"`
-	Encoding      string              `json:"encoding"`
-	ServiceType   string              `json:"service_type"`
-	Groups        []OrganizationGroup `json:"groups"`
-	ModelIpfsHash string              `json:"model_ipfs_hash"`
-	MpeAddress    string              `json:"mpe_address"`
+	Version          int                 `json:"version"`
+	DisplayName      string              `json:"display_name"`
+	Encoding         string              `json:"encoding"`
+	ServiceType      string              `json:"service_type"`
+	Groups           []OrganizationGroup `json:"groups"`
+	ModelIpfsHash    string              `json:"model_ipfs_hash"`
+	ServiceApiSource string              `json:"service_api_source"`
+	MpeAddress       string              `json:"mpe_address"`
 
 	multiPartyEscrowAddress common.Address
 	defaultPricing          Pricing
@@ -196,6 +196,7 @@ type Subscriptions struct {
 	IsActive     string         `json:"isActive"`
 	Subscription []Subscription `json:"subscription"`
 }
+
 type Tier struct {
 	Type            string      `json:"type"`
 	PlanName        string      `json:"planName"`
@@ -205,6 +206,7 @@ type Tier struct {
 	DetailsURL      string      `json:"detailsUrl"`
 	IsActive        string      `json:"isActive"`
 }
+
 type Licenses struct {
 	Subscriptions Subscriptions `json:"subscriptions,omitempty"`
 	Tiers         []Tier        `json:"tiers"`
@@ -252,7 +254,7 @@ func ServiceMetaData() *ServiceMetadata {
 	var err error
 	if config.GetBool(config.BlockchainEnabledKey) {
 		ipfsHash := string(getServiceMetaDataUrifromRegistry())
-		metadata, err = GetServiceMetaDataFromIPFS(FormatHash(ipfsHash))
+		metadata, err = GetServiceMetaDataFromIPFS(ipfsHash)
 		if err != nil {
 			zap.L().Panic("error on determining service metadata from file", zap.Error(err))
 		}
@@ -268,9 +270,7 @@ func ReadServiceMetaDataFromLocalFile(filename string) (*ServiceMetadata, error)
 	if err != nil {
 		return nil, errors.Wrapf(err, "could not read file: %v", filename)
 	}
-	strJson := string(file)
-	metadata, err := InitServiceMetaDataFromJson(strJson)
-
+	metadata, err := InitServiceMetaDataFromJson(file)
 	if err != nil {
 		return nil, fmt.Errorf("error reading local file service_metadata.json ")
 	}
@@ -317,13 +317,16 @@ func getServiceMetaDataUrifromRegistry() []byte {
 }
 
 func GetServiceMetaDataFromIPFS(hash string) (*ServiceMetadata, error) {
-	jsondata := ipfsutils.GetIpfsFile(hash)
+	jsondata, err := ipfsutils.ReadFile(hash)
+	if err != nil {
+		return nil, err
+	}
 	return InitServiceMetaDataFromJson(jsondata)
 }
 
-func InitServiceMetaDataFromJson(jsonData string) (*ServiceMetadata, error) {
+func InitServiceMetaDataFromJson(jsonData []byte) (*ServiceMetadata, error) {
 	metaData := new(ServiceMetadata)
-	err := json.Unmarshal([]byte(jsonData), &metaData)
+	err := json.Unmarshal(jsonData, &metaData)
 	if err != nil {
 		zap.L().Error(err.Error(), zap.Any("jsondata", jsonData))
 		return nil, err
@@ -350,7 +353,7 @@ func InitServiceMetaDataFromJson(jsonData string) (*ServiceMetadata, error) {
 		zap.L().Error(err.Error())
 	}
 
-	zap.L().Debug("Traning method", zap.String("json", string(trainingMethodsJson)))
+	zap.L().Debug("Training method", zap.String("json", string(trainingMethodsJson)))
 
 	return metaData, err
 }
@@ -488,8 +491,22 @@ func isElementInArray(a string, list []string) bool {
 func setServiceProto(metaData *ServiceMetadata) (err error) {
 	metaData.DynamicPriceMethodMapping = make(map[string]string, 0)
 	metaData.TrainingMethods = make([]string, 0)
-	//This is to handler the scenario where there could be multiple protos associated with the service proto
-	protoFiles, err := ipfsutils.ReadFilesCompressed(ipfsutils.GetIpfsFile(metaData.ModelIpfsHash))
+	var rawFile []byte
+
+	// for backwards compatibility
+	if metaData.ModelIpfsHash != "" {
+		rawFile, err = ipfsutils.GetIpfsFile(metaData.ServiceApiSource)
+	}
+
+	if metaData.ServiceApiSource != "" {
+		rawFile, err = ipfsutils.ReadFile(metaData.ServiceApiSource)
+	}
+
+	if err != nil {
+		zap.L().Error("Error in retrieving file from filecoin/ipfs", zap.Error(err))
+	}
+
+	protoFiles, err := ipfsutils.ReadFilesCompressed(rawFile)
 	if err != nil {
 		return err
 	}

blockchain/utils.go

@@ -3,9 +3,6 @@ package blockchain
 import (
 	"encoding/base64"
 	"fmt"
-	"regexp"
-	"strings"
-
 	"github.com/ethereum/go-ethereum/common"
 	"go.uber.org/zap"
 )
@@ -52,11 +49,6 @@ func StringToBytes32(str string) [32]byte {
 	return byte32
 }
 
-func RemoveSpecialCharactersfromHash(pString string) string {
-	reg := regexp.MustCompile("[^a-zA-Z0-9=]")
-	return reg.ReplaceAllString(pString, "")
-}
-
 func ConvertBase64Encoding(str string) ([32]byte, error) {
 	var byte32 [32]byte
 	data, err := base64.StdEncoding.DecodeString(str)
@@ -68,14 +60,6 @@ func ConvertBase64Encoding(str string) ([32]byte, error) {
 	return byte32, nil
 }
 
-func FormatHash(ipfsHash string) string {
-	zap.L().Debug("Before Formatting", zap.String("metadataHash", ipfsHash))
-	ipfsHash = strings.Replace(ipfsHash, IpfsPrefix, "", -1)
-	ipfsHash = RemoveSpecialCharactersfromHash(ipfsHash)
-	zap.L().Debug("After Formatting", zap.String("metadataUri", ipfsHash))
-	return ipfsHash
-}
-
 func ToChecksumAddress(hexAddress string) string {
 	address := common.HexToAddress(hexAddress)
 	mixedAddress := common.NewMixedcaseAddress(address)

blockchain/utils_test.go

@@ -11,49 +11,13 @@ func TestBytesToBase64(t *testing.T) {
 	assert.Equal(t, "AQL+/w==", base64)
 }
 
-func TestFormatHash(t *testing.T) {
-	s2 := []byte("ipfs://Here is a string....+=")
-	hash := FormatHash(string(s2))
-	assert.Equal(t, hash, "Hereisastring=")
-	s2 = []byte("QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp")
-
-	b4 := append(s2, make([]byte, 3)...)
-	assert.NotEqual(t, "QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp", string(b4))
-	assert.Equal(t, "QmaGnQ3iVZPuPwdam2rEeQcCSoCYRpxjnZhQ6Z2oeeRSrp", FormatHash(string(b4)))
-}
 func TestConvertBase64Encoding(t *testing.T) {
-
 	if _, err := ConvertBase64Encoding("n@@###zNEetD1kzU3PZqR4nHPS8erDkrUK0hN4iCBQ4vH5U"); err != nil {
 		assert.Equal(t, err.Error(), "illegal base64 data at input byte 1")
 	}
-
 }
 
 func TestToChecksumAddress(t *testing.T) {
-
 	assert.Equal(t, "0xE9D09A6C296ACDd4C01b21F407aC93FDfC63e78c", ToChecksumAddress("0xe9d09A6C296aCdd4c01b21f407ac93fdfC63E78C"))
-
 	assert.Equal(t, "0xE9D09A6C296ACDd4C01b21F407aC93FDfC63e78c", ToChecksumAddress("0xe9d09A6C296aCdd4c01b21f407ac93fdfC63E78C"))
 }
-
-func TestRemoveSpecialCharactersfromHash(t *testing.T) {
-	testCases := []struct {
-		input          string
-		expectedOutput string
-	}{
-		{"abc123", "abc123"},
-		{"abc123!@#", "abc123"},
-		{"a1b2c3 ~`!@#$%^&*()_+-={}[]|\\:;\"'<>,.?/", "a1b2c3="},
-		{"abc=123", "abc=123"},
-		{"a1!b2@c3#=4", "a1b2c3=4"},
-	}
-
-	for _, tc := range testCases {
-		t.Run(tc.input, func(t *testing.T) {
-			output := RemoveSpecialCharactersfromHash(tc.input)
-			if output != tc.expectedOutput {
-				t.Errorf("RemoveSpecialCharactersfromHash(%q) = %q; want %q", tc.input, output, tc.expectedOutput)
-			}
-		})
-	}
-}

go.mod

@@ -6,7 +6,7 @@ require (
 	github.com/OneOfOne/go-utils v0.0.0-20180319162427-6019ff89a94e
 	github.com/bufbuild/protocompile v0.14.1
 	github.com/emicklei/proto v1.13.2
-	github.com/ethereum/go-ethereum v1.14.9
+	github.com/ethereum/go-ethereum v1.14.11
 	github.com/golang-collections/collections v0.0.0-20130729185459-604e922904d3
 	github.com/golang-jwt/jwt/v5 v5.2.1
 	github.com/gorilla/handlers v1.5.2
@@ -30,7 +30,7 @@ require (
 	go.etcd.io/etcd/client/v3 v3.5.16
 	go.etcd.io/etcd/server/v3 v3.5.16
 	go.uber.org/zap v1.27.0
-	golang.org/x/crypto v0.27.0
+	golang.org/x/crypto v0.28.0
 	golang.org/x/net v0.29.0
 	golang.org/x/time v0.6.0
 	google.golang.org/grpc v1.66.2
@@ -56,7 +56,7 @@ require (
 	github.com/cockroachdb/pebble v1.1.2 // indirect
 	github.com/cockroachdb/redact v1.1.5 // indirect
 	github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06 // indirect
-	github.com/consensys/bavard v0.1.17 // indirect
+	github.com/consensys/bavard v0.1.22 // indirect
 	github.com/consensys/gnark-crypto v0.14.0 // indirect
 	github.com/coreos/go-semver v0.3.1 // indirect
 	github.com/coreos/go-systemd/v22 v22.5.0 // indirect
@@ -217,8 +217,8 @@ require (
 	golang.org/x/exp v0.0.0-20240719175910-8a7402abbf56 // indirect
 	golang.org/x/mod v0.19.0 // indirect
 	golang.org/x/sync v0.8.0 // indirect
-	golang.org/x/sys v0.25.0 // indirect
-	golang.org/x/text v0.18.0 // indirect
+	golang.org/x/sys v0.26.0 // indirect
+	golang.org/x/text v0.19.0 // indirect
 	golang.org/x/tools v0.23.0 // indirect
 	golang.org/x/xerrors v0.0.0-20240716161551-93cc26a95ae9 // indirect
 	gonum.org/v1/gonum v0.15.0 // indirect

go.sum

@@ -117,6 +117,8 @@ github.com/cockroachdb/tokenbucket v0.0.0-20230807174530-cc333fc44b06/go.mod h1:
 github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI=
 github.com/consensys/bavard v0.1.17 h1:53CdY/g35YSH9oRoa/b29tZinaiOEJYBmf9vydozPpE=
 github.com/consensys/bavard v0.1.17/go.mod h1:9ItSMtA/dXMAiL7BG6bqW2m3NdSEObYWoH223nGHukI=
+github.com/consensys/bavard v0.1.22 h1:Uw2CGvbXSZWhqK59X0VG/zOjpTFuOMcPLStrp1ihI0A=
+github.com/consensys/bavard v0.1.22/go.mod h1:k/zVjHHC4B+PQy1Pg7fgvG3ALicQw540Crag8qx+dZs=
 github.com/consensys/gnark-crypto v0.14.0 h1:DDBdl4HaBtdQsq/wfMwJvZNE80sHidrK3Nfrefatm0E=
 github.com/consensys/gnark-crypto v0.14.0/go.mod h1:CU4UijNPsHawiVGNxe9co07FkzCeWHHrb1li/n1XoU0=
 github.com/containerd/cgroups v1.1.0 h1:v8rEWFl6EoqHB+swVNjVoCJE8o3jX7e8nqBGPLaDFBM=
@@ -182,6 +184,10 @@ github.com/ethereum/c-kzg-4844 v1.0.3 h1:IEnbOHwjixW2cTvKRUlAAUOeleV7nNM/umJR+qy
 github.com/ethereum/c-kzg-4844 v1.0.3/go.mod h1:VewdlzQmpT5QSrVhbBuGoCdFJkpaJlO1aQputP83wc0=
 github.com/ethereum/go-ethereum v1.14.9 h1:J7iwXDrtUyE9FUjUYbd4c9tyzwMh6dTJsKzo9i6SrwA=
 github.com/ethereum/go-ethereum v1.14.9/go.mod h1:QeW+MtTpRdBEm2pUFoonByee8zfHv7kGp0wK0odvU1I=
+github.com/ethereum/go-ethereum v1.14.10 h1:kC24WjYeRjDy86LVo6MfF5Xs7nnUu+XG4AjaYIaZYko=
+github.com/ethereum/go-ethereum v1.14.10/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E=
+github.com/ethereum/go-ethereum v1.14.11 h1:8nFDCUUE67rPc6AKxFj7JKaOa2W/W1Rse3oS6LvvxEY=
+github.com/ethereum/go-ethereum v1.14.11/go.mod h1:+l/fr42Mma+xBnhefL/+z11/hcmJ2egl+ScIVPjhc7E=
 github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9 h1:8NfxH2iXvJ60YRB8ChToFTUzl8awsc3cJ8CbLjGIl/A=
 github.com/ethereum/go-verkle v0.1.1-0.20240829091221-dffa7562dbe9/go.mod h1:M3b90YRnzqKyyzBEWJGqj8Qff4IDeXnzFw0P9bFw3uk=
 github.com/facebookgo/atomicfile v0.0.0-20151019160806-2de1f203e7d5 h1:BBso6MBKW8ncyZLv37o+KNyy0HrrHgfnOaGQC2qvN+A=
@@ -1064,6 +1070,8 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh
 golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc=
 golang.org/x/crypto v0.27.0 h1:GXm2NjJrPaiv/h1tb2UH8QfgC/hOf/+z0p6PT8o1w7A=
 golang.org/x/crypto v0.27.0/go.mod h1:1Xngt8kV6Dvbssa53Ziq6Eqn0HqbZi5Z6R0ZpwQzt70=
+golang.org/x/crypto v0.28.0 h1:GBDwsMXVQi34v5CCYUm2jkJvu4cbtru2U4TN2PSyQnw=
+golang.org/x/crypto v0.28.0/go.mod h1:rmgy+3RHxRZMyY0jjAJShp2zgEdOqj2AO7U0pYmeQ7U=
 golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190306152737-a1d7652674e8/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA=
 golang.org/x/exp v0.0.0-20190510132918-efd6b22b2522/go.mod h1:ZjyILWgesfNpC6sMxTJOJm9Kp84zZh5NQWvqDGG3Qr8=
@@ -1213,6 +1221,8 @@ golang.org/x/sys v0.11.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.14.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/sys v0.25.0 h1:r+8e+loiHxRqhXVl6ML1nO3l1+oFoWbnlu2Ehimmi34=
 golang.org/x/sys v0.25.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
+golang.org/x/sys v0.26.0 h1:KHjCJyddX0LoSTb3J+vWpupP9p0oznkqVk/IfjymZbo=
+golang.org/x/sys v0.26.0/go.mod h1:/VUhepiaJMQUp4+oa/7Zr1D23ma6VTLIYjOOTFZPUcA=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8=
 golang.org/x/term v0.5.0/go.mod h1:jMB1sMXY+tzblOD4FWmEbocvup2/aLOaQEp7JmGp78k=
@@ -1226,6 +1236,8 @@ golang.org/x/text v0.3.7/go.mod h1:u+2+/6zg+i71rQMx5EYifcz6MCKuco9NR6JIITiCfzQ=
 golang.org/x/text v0.7.0/go.mod h1:mrYo+phRRbMaCq/xk9113O4dZlRixOauAjOtrjsXDZ8=
 golang.org/x/text v0.18.0 h1:XvMDiNzPAl0jr17s6W9lcaIhGUfUORdGCNsuLmPG224=
 golang.org/x/text v0.18.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
+golang.org/x/text v0.19.0 h1:kTxAhCbGbxhK0IwgSKiMO5awPoDQ0RpfiVYBfK860YM=
+golang.org/x/text v0.19.0/go.mod h1:BuEKDfySbSR4drPmRPG/7iBdf8hvFMuRexcpahXilzY=
 golang.org/x/time v0.0.0-20180412165947-fbb02b2291d2/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20181108054448-85acf8d2951c/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=
 golang.org/x/time v0.0.0-20190308202827-9d24e82272b4/go.mod h1:tRJNPiyCQ0inRvYxbN9jk5I+vvW/OXSQhTDSoE431IQ=

ipfsutils/common.go

@@ -0,0 +1,35 @@
+package ipfsutils
+
+import (
+	"go.uber.org/zap"
+	"regexp"
+	"strings"
+)
+
+const (
+	IpfsPrefix     = "ipfs://"
+	FilecoinPrefix = "filecoin://"
+)
+
+func ReadFile(hash string) (rawFile []byte, err error) {
+	if strings.HasPrefix(hash, FilecoinPrefix) {
+		rawFile, err = GetLighthouseFile(formatHash(hash))
+	} else {
+		rawFile, err = GetIpfsFile(formatHash(hash))
+	}
+	return rawFile, err
+}
+
+func formatHash(hash string) string {
+	zap.L().Debug("Before Formatting", zap.String("metadataHash", hash))
+	hash = strings.Replace(hash, IpfsPrefix, "", -1)
+	hash = strings.Replace(hash, FilecoinPrefix, "", -1)
+	hash = removeSpecialCharacters(hash)
+	zap.L().Debug("After Formatting", zap.String("metadataHash", hash))
+	return hash
+}
+
+func removeSpecialCharacters(pString string) string {
+	reg := regexp.MustCompile("[^a-zA-Z0-9=]")
+	return reg.ReplaceAllString(pString, "")
+}