Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

35 dev releasefailing #51

Open
wants to merge 4 commits into
base: dev
Choose a base branch
from
Open

35 dev releasefailing #51

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
4 commits
Select commit Hold shift + click to select a range
998ccc7
add more detailed logging of possible errors.
valentinedwv Sep 1, 2023
5d2a871
Merge branch 'dev' of https://github.com/gleanerio/nabu into 35_dev_r…
valentinedwv Sep 1, 2023
0958202
#52 if it's not there, add it.
valentinedwv Sep 30, 2023
4f3bc59
#52 if it's not there, add it.
valentinedwv Oct 5, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
23 changes: 15 additions & 8 deletions internal/objects/pipecopy.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -71,14 +71,16 @@ func PipeCopy(v1 *viper.Viper, mc *minio.Client, name, bucket, prefix, destprefi
for object := range objectCh {
fo, err := mc.GetObject(context.Background(), bucket, object.Key, minio.GetObjectOptions{})
if err != nil {
fmt.Println(err)
log.Errorf(" failed to read object %s %s ", object.Key, err)
//fmt.Println(err)
}

log.Tracef(" processing object %s ", object.Key)
var b bytes.Buffer
bw := bufio.NewWriter(&b)

_, err = io.Copy(bw, fo)
if err != nil {
log.Errorf(" failed to read object %s %s ", object.Key, err)
log.Println(err)
}

Expand All @@ -91,8 +93,8 @@ func PipeCopy(v1 *viper.Viper, mc *minio.Client, name, bucket, prefix, destprefi
} else {
nq, err = graph.JSONLDToNQ(v1, s)
if err != nil {
log.Println(err)
return
log.Errorf(" failed to convert to NQ %s %s ", object.Key, err)
continue
}
}

Expand All @@ -103,24 +105,28 @@ func PipeCopy(v1 *viper.Viper, mc *minio.Client, name, bucket, prefix, destprefi
} else {
snq, err = graph.Skolemization(nq, object.Key)
if err != nil {
return
log.Errorf(" failed Skolemization %s %s ", object.Key, err)
continue
}
}

// 1) get graph URI
ctx, err := graph.MakeURN(v1, object.Key)
if err != nil {
return
log.Errorf(" failed MakeURN %s %s ", object.Key, err)
continue
}
// 2) convert NT to NQ
csnq, err := graph.NtToNq(snq, ctx)
if err != nil {
return
log.Errorf(" failed NtToNq %s %s ", object.Key, err)
continue
}

_, err = pw.Write([]byte(csnq))
if err != nil {
return
log.Errorf(" failed pipe write %s %s ", object.Key, err)
continue
}
}
}()
Expand All @@ -131,6 +137,7 @@ func PipeCopy(v1 *viper.Viper, mc *minio.Client, name, bucket, prefix, destprefi
_, err := mc.PutObject(context.Background(), bucket, fmt.Sprintf("%s/%s", destprefix, name), pr, -1, minio.PutObjectOptions{})
//_, err := mc.PutObject(context.Background(), bucket, fmt.Sprintf("%s/%s", prefix, name), pr, -1, minio.PutObjectOptions{})
if err != nil {
log.Errorf(" failed PutObject bucket: %s %s/%s ", bucket, destprefix, name)
log.Println(err)
return
}
Expand Down
9 changes: 9 additions & 0 deletions pkg/config/implnet.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@ package config

import (
"fmt"
log "github.com/sirupsen/logrus"
"github.com/spf13/viper"
)

Expand All @@ -24,6 +25,14 @@ func GetImplNetworkConfig(viperConfig *viper.Viper) (ImplNetwork, error) {
// use config.Sub("gleaner)
func readImpleNetworkConfig(implementation_networkSubtress *viper.Viper) (ImplNetwork, error) {
var gleanerCfg ImplNetwork
if implementation_networkSubtress == nil {
// trace, otherwise goes off for every item
log.Trace("No Implementation Network in config file: Add \n implementation_network: \n orgname: NAME ")
implementation_networkSubtress = viper.New()
//for key, value := range implNetworkTemplate {
// implementation_networkSubtress.Set(key, value)
//}
}
for key, value := range implNetworkTemplate {
implementation_networkSubtress.SetDefault(key, value)
}
Expand Down