Skip to content

Commit

Permalink
Switch start and end tokens
Browse files Browse the repository at this point in the history 
Follow the standard regex notation
  • Loading branch information
@mb-14
mb-14 committed Nov 20, 2023
1 parent e6365fd commit 9cbdc8d
Show file tree
Hide file tree
Showing 2 changed files with 21 additions and 21 deletions.
4 changes: 2 additions & 2 deletions gomarkov.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,8 +12,8 @@ import (
// Tokens are wrapped around a sequence of words to maintain the
// start and end transition counts
const (
StartToken = "$"
EndToken = "^"
StartToken = "^"
EndToken = "$"
)

// Chain is a markov chain instance
Expand Down
38 changes: 19 additions & 19 deletions gomarkov_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,8 @@ func TestChain_MarshalJSON(t *testing.T) {
}{
{"Empty chain", 2, [][]string{}, `{"int":2,"spool_map":{},"freq_mat":{}}`, false},
{"Empty chain, order 1", 1, [][]string{}, `{"int":1,"spool_map":{},"freq_mat":{}}`, false},
{"Trained once", 1, [][]string{{"Test"}}, `{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`, false},
{"Trained on more data", 1, [][]string{{"test", "data"}, {"test", "data"}, {"test", "node"}}, `{"int":1,"spool_map":{"$":0,"^":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":1},"2":{"3":2},"4":{"3":1}}}`, false},
{"Trained once", 1, [][]string{{"Test"}}, `{"int":1,"spool_map":{"$":2,"Test":1,"^":0},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`, false},
{"Trained on more data", 1, [][]string{{"test", "data"}, {"test", "data"}, {"test", "node"}}, `{"int":1,"spool_map":{"$":3,"^":0,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":1},"2":{"3":2},"4":{"3":1}}}`, false},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
Expand Down Expand Up @@ -45,7 +45,7 @@ func TestChain_UnmarshalJSON(t *testing.T) {
wantErr bool
}{
{"Empty chain", []byte(`{"int":2,"spool_map":{},"freq_mat":{}}`), false},
{"More complex chain", []byte(`{"int":1,"spool_map":{"$":0,"^":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":1},"2":{"3":2},"4":{"3":1}}}`), false},
{"More complex chain", []byte(`{"int":1,"spool_map":{"^":0,"$":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":1},"2":{"3":2},"4":{"3":1}}}`), false},
{"Invalid json", []byte(`{{"int":2,"spool_map":{},"freq_mat":{}}`), true},
}
for _, tt := range tests {
Expand Down Expand Up @@ -113,42 +113,42 @@ func TestChain_TransitionProbability(t *testing.T) {
}{
{
"Simple transition positive",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{next: "Test", current: NGram{"$"}},
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{next: "Test", current: NGram{"^"}},
1,
false,
},
{
"Simple transition negative",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{next: "Test", current: NGram{"Test"}},
0,
false,
},
{
"Unknown next Ngram",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{next: "Unknown", current: NGram{"Test"}},
0,
false,
},
{
"Unknown ncurent Ngram",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{next: "Test", current: NGram{"Unknown"}},
0,
false,
},
{
"Invalid Ngram",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{next: "Unknown", current: NGram{"Test", "data"}},
0,
true,
},
{
"More than 1 option",
[]byte(`{"int":1,"spool_map":{"$":0,"^":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":2},"2":{"3":2},"4":{"3":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"$":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":2},"2":{"3":2},"4":{"3":1}}}`),
args{next: "node", current: NGram{"test"}},
0.5,
false,
Expand Down Expand Up @@ -184,43 +184,43 @@ func TestChain_Generate(t *testing.T) {
}{
{
"Start of simple chain",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"$"}},
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"^"}},
"Test",
false,
},
{
"End of simple chain",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"Test"}},
"^",
"$",
false,
},
{
"Complex chain",
[]byte(`{"int":1,"spool_map":{"$":0,"^":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":0},"2":{"3":2},"4":{"3":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"$":3,"data":2,"node":4,"test":1},"freq_mat":{"0":{"1":3},"1":{"2":2,"4":0},"2":{"3":2},"4":{"3":1}}}`),
args{current: NGram{"test"}},
"data",
false,
},
{
"Invalid Ngram",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"Invalid", "Ngram"}},
"",
true,
},
{
"Unknown Ngram",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"Unknown"}},
"",
true,
},
{
"No next state",
[]byte(`{"int":1,"spool_map":{"$":0,"Test":1,"^":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"^"}},
[]byte(`{"int":1,"spool_map":{"^":0,"Test":1,"$":2},"freq_mat":{"0":{"1":1},"1":{"2":1}}}`),
args{current: NGram{"$"}},
"",
false,
},
Expand Down

0 comments on commit 9cbdc8d

Please sign in to comment.