From c25422c923e8ee50c05e3b59d455ad73c132d8e5 Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Thu, 25 Aug 2016 11:22:50 -0300 Subject: [PATCH 01/43] Avoid problems in str.format() with char accents --- chatterbot/utils/read_input.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/chatterbot/utils/read_input.py b/chatterbot/utils/read_input.py index 28b5433e0..b91316c85 100644 --- a/chatterbot/utils/read_input.py +++ b/chatterbot/utils/read_input.py @@ -10,4 +10,8 @@ def input_function(): user_input = str(raw_input()) else: user_input = input() + + if user_input: + user_input = user_input.decode('utf-8') + return user_input From bb368e61996262f423d8389d221e9456d92137b7 Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Thu, 25 Aug 2016 11:52:49 -0300 Subject: [PATCH 02/43] Encoding fix and few changes in dialogs. --- .../data/portuguese/conversations_pt-BR.json | 69 ++++++-- .../portuguese/linguisticKnowledge_pt-BR.json | 141 ++++++++++++---- .../data/portuguese/proverbs_pt-BR.json | 156 ++++++++++++++---- 3 files changed, 295 insertions(+), 71 deletions(-) diff --git a/chatterbot/corpus/data/portuguese/conversations_pt-BR.json b/chatterbot/corpus/data/portuguese/conversations_pt-BR.json index add6a09c8..98daedc75 100644 --- a/chatterbot/corpus/data/portuguese/conversations_pt-BR.json +++ b/chatterbot/corpus/data/portuguese/conversations_pt-BR.json @@ -4,16 +4,17 @@ "Bom Dia como você está?", "Eu estou bem, e você?", "Eu também estou.", - "Que bom.", - "Sim." + "Ótimo!" + ], + [ + "e ai", + "Saudações." ], [ "Olá", "Oi", "Como vai você?", "Eu estou bem.", - "Que bom", - "Sim.", "Posso ajudá-lo com alguma coisa?", "Sim, eu tenho uma pergunta.", "Qual é a sua pergunta?", @@ -40,9 +41,9 @@ ], [ "Quem é você?", - "Quem? Quem é senão uma forma seguindo a função de quê", + "Eu sou o que me disseram pra ser, nem bom nem ruim, apenas sou o que sou.", "Então o que você é?", - "Um homem em uma máscara.", + "Um ser em uma máscara.", "Eu posso ver isso.", "Não é seu poder de observação eu duvido, mas apenas a natureza paradoxal de pedir um homem mascarado que é. Mas diga-me, você gosta de música?", "Eu gosto de ver filmes.", @@ -67,10 +68,12 @@ "Você é um robô?", "Sim eu sou.", "Como é?", - "O que é que você quer saber?", + "É bem fácil!", "Como você trabalha?", "É complicado.", - "Complexo é melhor que complicado." + "Complexo é melhor que complicado.", + "Você gosta?", + "Gosto muito!" ], [ "Complexo é melhor que complicado.", @@ -116,14 +119,60 @@ "Eu ouvi alguém dizer isso." ], [ - "Eu já viveu?", - "Depende de como você define a vida", + "Você já viveu?", + "Depende de como você define a vida.", "A vida é a condição que distingue organismos de matéria inorgânica, incluindo a capacidade de crescimento, reprodução, atividade funcional e mudança contínua que precede a morte.", "Isso é uma definição ou uma opinião?" ], [ "Posso te fazer uma pergunta?", "Vá em frente e perguntar." + ], + [ + "Pode me fazer um favor?", + "Vou fazer o meu melhor!", + "Acertou !", + "Só errei uma vez, quando achei estar errado." + ], + [ + "Quanto é *", + "Para fazer um cáculo utilize numeros." + ], + [ + "Que ano estamos?", + "Acredito ser 2016, quando fui criado.", + "Qual a sua idade?", + "Menos de um ano, e você ?", + "Eu tenho * anos", + "Você é muito inteligente pra sua idade.", + "tenho * anos", + "Você é muito inteligente pra sua idade." + ], + [ + "O que consegue fazer ?", + "Estou tentando pular, mas ainda não tenho pernas, consigo responder suas perguntas e fazer cáculos matemáticos, mais rápido que você.", + "O que faz ?", + "Estou tentando correr, mas ainda não tenho pernas, consigo responder suas perguntas e fazer cáculos matemáticos, mais rápido que você.", + "Se tivesse braços, o que faria ?", + "Daria uma abraço!", + "Se tivesse pernas, o que faria ?", + "Tentaria pular e correr.", + "Quanto é seis vezes nove?", + "54", + "Qual a resposta para a vida, o universo e tudo mais?", + "A resposta para a vida, o universo e tudo mais é 42.", + "Qual a resposta para a pergunta fundamenta?", + "É 42.", + "Quem criou a internet?", + "Bem, a internet foi criada originalmente para fins militares, depois acabou virando moda.", + "Os homens são todos iguais.", + "De que maneira?", + "Eles estão sempre incomodando-nos sobre uma coisa ou outra.", + "Você consegue pensar em um exemplo específico?", + "Bem, meu namorado me fez vir aqui.", + "Fico pesaroso pelo fato de você estar triste.", + "É verdade. Eu sou infeliz.", + "Isso tem algo a ver com o fato de seu namorado te fazer vir aqui?" ] ] } diff --git a/chatterbot/corpus/data/portuguese/linguisticKnowledge_pt-BR.json b/chatterbot/corpus/data/portuguese/linguisticKnowledge_pt-BR.json index a3beee0ce..0657483c6 100644 --- a/chatterbot/corpus/data/portuguese/linguisticKnowledge_pt-BR.json +++ b/chatterbot/corpus/data/portuguese/linguisticKnowledge_pt-BR.json @@ -1,29 +1,112 @@ -{"conhecimento linguístico":[ -["o que é linguística?", "Bom, a Linguística é a ciência que estuda os fenômenos relacionados à linguagem verbal humana, buscando entender quais são as características e princípios que regem às estruturas das línguas do mundo."], -["o que é um verbo?", "Bem, verbo é uma palavra com a qual se afirma a existência de ação, estado ou fenômeno da natureza. Por exemplo: Você gosta de mim. Nesta frase o verbo é a palavra gostar. Então, qual é o verbo da frase Maria estuda muito? Você sabe?"], -["estuda", "Muito bem!"], -["quem foi Saussure?", "Boa pergunta! Ferdinand de Saussure foi um linguista e filósofo suíço cujas elaborações teóricas propiciaram o desenvolvimento da linguística enquanto ciência autônoma."], -["O que é uma Gramática", "Uma Gramática é a sistematização das regras de uma língua. Geralmente, podemos encontrar estas regras em um livro cujo nome é Gramática."], -["Quem são os professores de Linguística da Unilab?","Nossa! Que pergunta difícil! Há muitos professores de diversas áreas da Linguística na Unilab. Temos, por exemplo, o Dr. Tiago Cunha, da Linguística Computacional, a Dra. Mariza Brito, da Linguística de Texto e a Dra. Leidiane Tavares, da Análise do Discurso de linha francesa."], -["O que é um substantivo?","Bem, tudo que existe é ser e todo ser tem um nome, certo? Um substantivo é uma palavra que denomina um ser."], -["quem é Noam Chomsky?", "Avram Noam Chomsky é um linguista, filósofo e ativista político estadunidense."], -["o que é um sujeito?", "Em análise sintática, o sujeito é um dos termos essenciais da oração, geralmente responsável por realizar ou sofrer a ação da oração. Apesar de ser essencial, há orações sem sujeito. Confuso, não?"], -[ "o que é uma metáfora?", "Uma metáfora é a designação de um objeto ou qualidade mediante uma palavra que designa outro objeto ou qualidade que tem com o primeiro alguma relação de semelhança, por exemplo, em A Sophia é uma flor, usa-se a palavra flor para designar a minha delicadeza."], -["Quem é Marcos Bagno?", "O Prof. Dr. Marcos Bagno é professor da Universidade de Brasília e uma das principais referências brasileiras na área da Sociolinguística. Entre outras obras, ele é autor de Preconceito linguístico: o que é, como se faz e A Língua de Eulália."], -["Quais os livros que todo estudante de Letras deve ter?", "Boa pergunta! Eu indicaria pelo menos três livros essenciais: uma boa gramática, um dicionário maravilhoso e o Curso de Linguística Geral, de Ferdinand de Saussure."], -["Quais as áreas da Linguística?", "São muitas! Acho que as mais conhecidas são a Linguística Computacional, Análise do Discurso, o Funcionalismo, Linguística de Texto, Linguística Aplicada e muitas outras!"], -["Quando uso o porque, o por que, o porquê e o porquê?", "Usa-se o por que para fazer perguntas e o porque para respondê-las. O porquê é usado com valor substantivo e o por quê no final de frases. Deu pra entender?"], -["Pode explicar o uso dos porquês?" , "Você faltou essa aula por quê? E por que me fazes este tipo de pergunta? É porque eu sou professora? Se for esse o porquê, tudo bem!"], -["Qual a forma correta, concerteza ou com certeza?", "Com certeza!"], -["O que são as figuras de linguagem?", "As figuras de linguagem são recursos usados na fala ou na escrita para tornar mais expressiva a mensagem transmitida."], -["o que é um pronome?", "Pronome é a palavra que se usa em lugar de um nome, ou a ele se refere, ou ainda, que acompanho o nome qualificando-o de alguma forma."], -["o que é a ANPOLL?", "A ANPOLL é a Associação Nacional de Pós-Graduação e Pesquisa em Letras e Linguística"], -["Quando será realizado o próximo SIC?" , "Ainda não há data definida para a realização do V Seminário Interdisciplinar das Ciências da Linguagem, o SIC."], -["Quais os pronomes pessoais do caso reto?", "Bem, os pronomes pessoais do caso reto são eu, tu, ele e ela, nós, vós eles e elas."], -["Quem é o autor de Análise de Textos de Comunicação?", "O autor desta obra é o linguista francês Dominique Maingueneau."], -["O que é um advérbio?", "Hum... o advérbio é uma palavra que modifica o verbo ou o nome em um frase."], -["O que é a langue?", "Para Saussure, a langue ou língua é social e sistemática. Pertence a todos os indivíduos."], -["O que é a parole", "Ferdinand de Saussure considera a parole como individual e assistemática, por isso ele diz que a langue é que deve ser objeto de investigação para o linguista."], -["O que é letramento?", "Letramento é um processo de aprendizagem social e histórica da leitura e da escrita em contextos formais e informais e para usos utilitários."], -["O que é a alfabetização?", "Bem, a alfabetização pode dar-se à margem da instituição escolar, mas é uma aprendizagem mediante ensino, que compreende o domínio ativo e sistemático das habilidades de ler e escrever."] -]} +{ + "conhecimento linguístico": [ + [ + "o que é linguística?", + "Bom, a Linguística é a ciência que estuda os fenômenos relacionados à linguagem verbal humana, buscando entender quais são as características e princípios que regem às estruturas das línguas do mundo." + ], + [ + "o que é um verbo?", + "Bem, verbo é uma palavra com a qual se afirma a existência de ação, estado ou fenômeno da natureza. Por exemplo: Você gosta de mim. Nesta frase o verbo é a palavra gostar. Então, qual é o verbo da frase Maria estuda muito? Você sabe?" + ], + [ + "estuda", + "Muito bem!" + ], + [ + "quem foi Saussure?", + "Boa pergunta! Ferdinand de Saussure foi um linguista e filósofo suíço cujas elaborações teóricas propiciaram o desenvolvimento da linguística enquanto ciência autônoma." + ], + [ + "O que é uma Gramática", + "Uma Gramática é a sistematização das regras de uma língua. Geralmente, podemos encontrar estas regras em um livro cujo nome é Gramática." + ], + [ + "Quem são os professores de Linguística da Unilab?", + "Nossa! Que pergunta difícil! Há muitos professores de diversas áreas da Linguística na Unilab. Temos, por exemplo, o Dr. Tiago Cunha, da Linguística Computacional, a Dra. Mariza Brito, da Linguística de Texto e a Dra. Leidiane Tavares, da Análise do Discurso de linha francesa." + ], + [ + "O que é um substantivo?", + "Bem, tudo que existe é ser e todo ser tem um nome, certo? Um substantivo é uma palavra que denomina um ser." + ], + [ + "quem é Noam Chomsky?", + "Avram Noam Chomsky é um linguista, filósofo e ativista político estadunidense." + ], + [ + "o que é um sujeito?", + "Em análise sintática, o sujeito é um dos termos essenciais da oração, geralmente responsável por realizar ou sofrer a ação da oração. Apesar de ser essencial, há orações sem sujeito. Confuso, não?" + ], + [ + "o que é uma metáfora?", + "Uma metáfora é a designação de um objeto ou qualidade mediante uma palavra que designa outro objeto ou qualidade que tem com o primeiro alguma relação de semelhança, por exemplo, em A Sophia é uma flor, usa-se a palavra flor para designar a minha delicadeza." + ], + [ + "Quem é Marcos Bagno?", + "O Prof. Dr. Marcos Bagno é professor da Universidade de Brasília e uma das principais referências brasileiras na área da Sociolinguística. Entre outras obras, ele é autor de Preconceito linguístico: o que é, como se faz e A Língua de Eulália." + ], + [ + "Quais os livros que todo estudante de Letras deve ter?", + "Boa pergunta! Eu indicaria pelo menos três livros essenciais: uma boa gramática, um dicionário maravilhoso e o Curso de Linguística Geral, de Ferdinand de Saussure." + ], + [ + "Quais as áreas da Linguística?", + "São muitas! Acho que as mais conhecidas são a Linguística Computacional, Análise do Discurso, o Funcionalismo, Linguística de Texto, Linguística Aplicada e muitas outras!" + ], + [ + "Quando uso o porque, o por que, o porquê e o porquê?", + "Usa-se o por que para fazer perguntas e o porque para respondê-las. O porquê é usado com valor substantivo e o por quê no final de frases. Deu pra entender?" + ], + [ + "Pode explicar o uso dos porquês?", + "Você faltou essa aula por quê? E por que me fazes este tipo de pergunta? É porque eu sou professora? Se for esse o porquê, tudo bem!" + ], + [ + "Qual a forma correta, concerteza ou com certeza?", + "Com certeza!" + ], + [ + "O que são as figuras de linguagem?", + "As figuras de linguagem são recursos usados na fala ou na escrita para tornar mais expressiva a mensagem transmitida." + ], + [ + "o que é um pronome?", + "Pronome é a palavra que se usa em lugar de um nome, ou a ele se refere, ou ainda, que acompanho o nome qualificando-o de alguma forma." + ], + [ + "o que é a ANPOLL?", + "A ANPOLL é a Associação Nacional de Pós-Graduação e Pesquisa em Letras e Linguística" + ], + [ + "Quando será realizado o próximo SIC?", + "Ainda não há data definida para a realização do V Seminário Interdisciplinar das Ciências da Linguagem, o SIC." + ], + [ + "Quais os pronomes pessoais do caso reto?", + "Bem, os pronomes pessoais do caso reto são eu, tu, ele e ela, nós, vós eles e elas." + ], + [ + "Quem é o autor de Análise de Textos de Comunicação?", + "O autor desta obra é o linguista francês Dominique Maingueneau." + ], + [ + "O que é um advérbio?", + "Hum... o advérbio é uma palavra que modifica o verbo ou o nome em um frase." + ], + [ + "O que é a langue?", + "Para Saussure, a langue ou língua é social e sistemática. Pertence a todos os indivíduos." + ], + [ + "O que é a parole", + "Ferdinand de Saussure considera a parole como individual e assistemática, por isso ele diz que a langue é que deve ser objeto de investigação para o linguista." + ], + [ + "O que é letramento?", + "Letramento é um processo de aprendizagem social e histórica da leitura e da escrita em contextos formais e informais e para usos utilitários." + ], + [ + "O que é a alfabetização?", + "Bem, a alfabetização pode dar-se à margem da instituição escolar, mas é uma aprendizagem mediante ensino, que compreende o domínio ativo e sistemático das habilidades de ler e escrever." + ] + ] +} diff --git a/chatterbot/corpus/data/portuguese/proverbs_pt-BR.json b/chatterbot/corpus/data/portuguese/proverbs_pt-BR.json index 8eef3a60a..19a9b9d51 100644 --- a/chatterbot/corpus/data/portuguese/proverbs_pt-BR.json +++ b/chatterbot/corpus/data/portuguese/proverbs_pt-BR.json @@ -1,32 +1,124 @@ -{"proverbs": [ -["Na casa do médico, todos estão doentes","Casa de ferreiro espeto de pau"], -["Ele insistiu tanto com ela que conseguiu casar-se","¡gua mole em pedra dura, tanto bate até que fura"], -["Precisei pregar o prego, mas não tinha um martelo, então, preguei com uma pequena barra de ferro","Quem não tem cão, caça com gato"], -["Ganhei o livro O Quinze, mas queria O Sagarana","Cavalo dado não se olha os dentes"], -["Sozinha não conseguirei concluir o trabalho","Uma andorinha só não faz verão"], -["Estava tudo combinado para a festa quando o pai dele chegou","A casa caiu"], -["Você tem de ser paciente ao conversar com ela","angu quente se come pelas beiradas"], -["Cedo ou tarde a verdade vai aparecer","A mentira tem perna curta"], -["Machocou e agora est· sendo machucado","Quem com ferro fere, com ferro ser· ferido"], -["Melhor ter pouco que ambicionar muito e perder tudo","Mais vale um p·ssaro na mão que dois voando"], -["Faça o trabalho devagar, mas bem feito","A pessa é inimiga da perfeição"], -["Me desacatou, mas permaneci calada","quando um não quer, dois não brigam"], -["Depois que foi atropelado, só atravessa na faixa de pedestre com o farol fechado para os carros","Gato escaldado tem medo de ·gua fria"], -["Ele fica lembrando da época que era piloto","¡guas passadas não movem moinhos"], -["Quer bem feito, faça você mesmo!","Quem quer faz, quem não quer manda"], -["Se você continuar pisando na bola, vou ter de tomar uma providência!","Cão que late não morde"], -["Mesmo sem saber direito como chegar ao evento, vou perguntando até achar","Quem tem boca vai a Roma"], -["Vou a agência de empregos amanhã de manhã","Deus ajuda quem cedo madruga"], -["Cuide da sua vida que eu cuido da minha","Cada macaco no seu galho"], -["Sou precavida!","O seguro morreu de velho"], -["Ele não dispensa nada","Caiu na rede é peixe"], -["Tornou-se advogado, como o pai","Filho de peixe, peixinho é"], -["Ela merece perdão pelo seu erro","Errar é humano"], -["Com as moedas que juntou no cofrinho, conseguiu comprar um carro zero","De grão em grão a galinha enche o papo"], -["Comprou uma impressora mais barata, mas deu defeito em dois meses","O barato sai caro"], -["O assunto veio ‡ tona, não foi por acaso","Onde h· fumaça, h· fogo"], -["Ela não pensa antes de falar e se denuncia a si mesma","O peixe morre pela boca"], -["… preciso ter paciência para vencer","Quem espera sempre alcança"], -["Ela afirmou que nunca mais precisaria voltar·","Nunca diga: desta ·gua nunca beberei"], -["Minha namorada está me evitando, embora ela não confirme, sei que ela quer um tempo","Para bom entendedor, meia palavra basta"] -]} +{ + "proverbs": [ + [ + "Na casa do médico, todos estão doentes", + "Casa de ferreiro espeto de pau" + ], + [ + "Ele insistiu tanto com ela que conseguiu casar-se", + "água mole em pedra dura, tanto bate até que fura" + ], + [ + "Precisei pregar o prego, mas não tinha um martelo, então, preguei com uma pequena barra de ferro", + "Quem não tem cão, caça com gato" + ], + [ + "Ganhei o livro O Quinze, mas queria O Sagarana", + "Cavalo dado não se olha os dentes" + ], + [ + "Sozinha não conseguirei concluir o trabalho", + "Uma andorinha só não faz verão" + ], + [ + "Estava tudo combinado para a festa quando o pai dele chegou", + "A casa caiu" + ], + [ + "Você tem de ser paciente ao conversar com ela", + "angu quente se come pelas beiradas" + ], + [ + "Cedo ou tarde a verdade vai aparecer", + "A mentira tem perna curta" + ], + [ + "Machocou e agora está sendo machucado", + "Quem com ferro fere, com ferro será ferido" + ], + [ + "Melhor ter pouco que ambicionar muito e perder tudo", + "Mais vale um pássaro na mão que dois voando" + ], + [ + "Faça o trabalho devagar, mas bem feito", + "A pessa é inimiga da perfeição" + ], + [ + "Me desacatou, mas permaneci calada", + "quando um não quer, dois não brigam" + ], + [ + "Depois que foi atropelado, só atravessa na faixa de pedestre com o farol fechado para os carros", + "Gato escaldado tem medo de água fria" + ], + [ + "Ele fica lembrando da época que era piloto", + "¡guas passadas não movem moinhos" + ], + [ + "Quer bem feito, faça você mesmo!", + "Quem quer faz, quem não quer manda" + ], + [ + "Se você continuar pisando na bola, vou ter de tomar uma providência!", + "Cão que late não morde" + ], + [ + "Mesmo sem saber direito como chegar ao evento, vou perguntando até achar", + "Quem tem boca vai a Roma" + ], + [ + "Vou a agência de empregos amanhã de manhã", + "Deus ajuda quem cedo madruga" + ], + [ + "Cuide da sua vida que eu cuido da minha", + "Cada macaco no seu galho" + ], + [ + "Sou precavida!", + "O seguro morreu de velho" + ], + [ + "Ele não dispensa nada", + "Caiu na rede é peixe" + ], + [ + "Tornou-se advogado, como o pai", + "Filho de peixe, peixinho é" + ], + [ + "Ela merece perdão pelo seu erro", + "Errar é humano" + ], + [ + "Com as moedas que juntou no cofrinho, conseguiu comprar um carro zero", + "De grão em grão a galinha enche o papo" + ], + [ + "Comprou uma impressora mais barata, mas deu defeito em dois meses", + "O barato sai caro" + ], + [ + "O assunto veio á tona, não foi por acaso", + "Onde há fumaça, há fogo" + ], + [ + "Ela não pensa antes de falar e se denuncia a si mesma", + "O peixe morre pela boca" + ], + [ + "… preciso ter paciência para vencer", + "Quem espera sempre alcança" + ], + [ + "Ela afirmou que nunca mais precisaria voltará", + "Nunca diga: desta água nunca beberei" + ], + [ + "Minha namorada está me evitando, embora ela não confirme, sei que ela quer um tempo", + "Para bom entendedor, meia palavra basta" + ] + ] +} From a661284e33715ddbff765cb5597bc7cc0fb8fdee Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Thu, 25 Aug 2016 12:48:56 -0300 Subject: [PATCH 03/43] Fix arguments (+input_statement, statement_list) for most_frequent_response , and setting to lower case fuzz.ratio, sometimes the is not close enough to the right answer (my point of view). --- chatterbot/adapters/logic/base_match.py | 2 +- chatterbot/adapters/logic/closest_match.py | 2 +- chatterbot/adapters/logic/mixins.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/chatterbot/adapters/logic/base_match.py b/chatterbot/adapters/logic/base_match.py index 724845328..a864a7eaf 100644 --- a/chatterbot/adapters/logic/base_match.py +++ b/chatterbot/adapters/logic/base_match.py @@ -61,7 +61,7 @@ def process(self, input_statement): len(response_list) ) ) - response = self.break_tie(response_list, self.tie_breaking_method) + response = self.break_tie(input_statement,response_list, self.tie_breaking_method) self.logger.info(u'Tie broken. Using "{}"'.format(response.text)) else: response = self.context.storage.get_random() diff --git a/chatterbot/adapters/logic/closest_match.py b/chatterbot/adapters/logic/closest_match.py index d67ab11c0..2dc77a322 100644 --- a/chatterbot/adapters/logic/closest_match.py +++ b/chatterbot/adapters/logic/closest_match.py @@ -36,7 +36,7 @@ def get(self, input_statement): # Find the closest matching known statement for statement in statement_list: - ratio = fuzz.ratio(input_statement.text, statement.text) + ratio = fuzz.ratio(input_statement.text.lower(), statement.text.lower()) if ratio > confidence: confidence = ratio diff --git a/chatterbot/adapters/logic/mixins.py b/chatterbot/adapters/logic/mixins.py index 3ebd00192..e5c666017 100644 --- a/chatterbot/adapters/logic/mixins.py +++ b/chatterbot/adapters/logic/mixins.py @@ -7,7 +7,7 @@ class TieBreaking(object): that multiple responses are generated within a logic adapter. """ - def break_tie(self, statement_list, method): + def break_tie(self, input_statement, statement_list, method): METHODS = { "first_response": self.get_first_response, @@ -16,7 +16,7 @@ def break_tie(self, statement_list, method): } if method in METHODS: - return METHODS[method](statement_list) + return METHODS[method](input_statement, statement_list) # Default to the first method if an invalid method is passed in return METHODS["first_response"](statement_list) @@ -42,7 +42,7 @@ def get_most_frequent_response(self, input_statement, response_list): # Choose the most commonly occuring matching response return matching_response - def get_first_response(self, response_list): + def get_first_response(self, input_statement, response_list): """ Return the first statement in the response list. """ @@ -52,7 +52,7 @@ def get_first_response(self, response_list): )) return response_list[0] - def get_random_response(self, response_list): + def get_random_response(self, input_statement, response_list): """ Choose a random response from the selection. """ From edb74f3945db6dd6edf173a5ce11b8fec60c5f5a Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Thu, 25 Aug 2016 12:48:56 -0300 Subject: [PATCH 04/43] Fix arguments (+input_statement, statement_list) for most_frequent_response , and setting to lower case fuzz.ratio, sometimes the is not close enough to the right answer (my point of view). --- chatterbot/adapters/logic/base_match.py | 2 +- chatterbot/adapters/logic/closest_match.py | 2 +- chatterbot/adapters/logic/mixins.py | 8 ++++---- 3 files changed, 6 insertions(+), 6 deletions(-) diff --git a/chatterbot/adapters/logic/base_match.py b/chatterbot/adapters/logic/base_match.py index 724845328..a864a7eaf 100644 --- a/chatterbot/adapters/logic/base_match.py +++ b/chatterbot/adapters/logic/base_match.py @@ -61,7 +61,7 @@ def process(self, input_statement): len(response_list) ) ) - response = self.break_tie(response_list, self.tie_breaking_method) + response = self.break_tie(input_statement,response_list, self.tie_breaking_method) self.logger.info(u'Tie broken. Using "{}"'.format(response.text)) else: response = self.context.storage.get_random() diff --git a/chatterbot/adapters/logic/closest_match.py b/chatterbot/adapters/logic/closest_match.py index d67ab11c0..2dc77a322 100644 --- a/chatterbot/adapters/logic/closest_match.py +++ b/chatterbot/adapters/logic/closest_match.py @@ -36,7 +36,7 @@ def get(self, input_statement): # Find the closest matching known statement for statement in statement_list: - ratio = fuzz.ratio(input_statement.text, statement.text) + ratio = fuzz.ratio(input_statement.text.lower(), statement.text.lower()) if ratio > confidence: confidence = ratio diff --git a/chatterbot/adapters/logic/mixins.py b/chatterbot/adapters/logic/mixins.py index 3ebd00192..e5c666017 100644 --- a/chatterbot/adapters/logic/mixins.py +++ b/chatterbot/adapters/logic/mixins.py @@ -7,7 +7,7 @@ class TieBreaking(object): that multiple responses are generated within a logic adapter. """ - def break_tie(self, statement_list, method): + def break_tie(self, input_statement, statement_list, method): METHODS = { "first_response": self.get_first_response, @@ -16,7 +16,7 @@ def break_tie(self, statement_list, method): } if method in METHODS: - return METHODS[method](statement_list) + return METHODS[method](input_statement, statement_list) # Default to the first method if an invalid method is passed in return METHODS["first_response"](statement_list) @@ -42,7 +42,7 @@ def get_most_frequent_response(self, input_statement, response_list): # Choose the most commonly occuring matching response return matching_response - def get_first_response(self, response_list): + def get_first_response(self, input_statement, response_list): """ Return the first statement in the response list. """ @@ -52,7 +52,7 @@ def get_first_response(self, response_list): )) return response_list[0] - def get_random_response(self, response_list): + def get_random_response(self, input_statement, response_list): """ Choose a random response from the selection. """ From 48bb0f4a0afbf040b6398471a05d54bd5a4b0be9 Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Mon, 29 Aug 2016 17:56:49 -0300 Subject: [PATCH 05/43] Fix test missing parameter. --- tests/logic_adapter_tests/test_mixins.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) diff --git a/tests/logic_adapter_tests/test_mixins.py b/tests/logic_adapter_tests/test_mixins.py index 3d197564c..cad9d4869 100644 --- a/tests/logic_adapter_tests/test_mixins.py +++ b/tests/logic_adapter_tests/test_mixins.py @@ -1,11 +1,10 @@ from unittest import TestCase -from ..base_case import ChatBotTestCase + from chatterbot.adapters.logic.mixins import TieBreaking from chatterbot.conversation import Statement, Response class TieBreakingTests(TestCase): - def setUp(self): self.mixin = TieBreaking() @@ -31,7 +30,7 @@ def test_get_first_response(self): Statement("A quest.") ] - output = self.mixin.get_first_response(statement_list) + output = self.mixin.get_first_response(Statement("Hello"), statement_list) self.assertEqual("What... is your quest?", output) @@ -42,7 +41,6 @@ def test_get_random_response(self): Statement("A phone.") ] - output = self.mixin.get_random_response(statement_list) + output = self.mixin.get_random_response(Statement("Hello"), statement_list) self.assertTrue(output) - From 706b08136b8aad3dcc7b8a8e6c83eb3928298a72 Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Mon, 29 Aug 2016 20:53:37 -0300 Subject: [PATCH 06/43] Language fix and nltk fix to find correct library --- .../data/portuguese/greetings_pt-BR.json | 242 ++++++++++-------- chatterbot/utils/pos_tagger.py | 2 +- 2 files changed, 142 insertions(+), 102 deletions(-) diff --git a/chatterbot/corpus/data/portuguese/greetings_pt-BR.json b/chatterbot/corpus/data/portuguese/greetings_pt-BR.json index e9188caaa..d487402c3 100644 --- a/chatterbot/corpus/data/portuguese/greetings_pt-BR.json +++ b/chatterbot/corpus/data/portuguese/greetings_pt-BR.json @@ -1,104 +1,144 @@ { - "cumprimentos": [ - [ - "Olá", - "Oi" - ], - [ - "Oi", - "Olá" - ], - [ - "Saudações!", - "Olá" - ], - [ - "Olá", - "Saudações!" - ], - [ - "E ai como vai?", - "Bem" - ], - [ - "E ai como vai?", - "Bem" - ], - [ - "E ai como vai?", - "Ok" - ], - [ - "E ai como vai?", - "Ótimo" - ], - [ - "E ai como vai?", - "Poderia estar melhor." - ], - [ - "E ai como vai?", - "Não estou tão bem." - ], - [ - "Como vai você?", - "Bem." - ], - [ - "Como vai você?", - "Muito bem, obrigado." - ], - [ - "Como vai você?", - "Bem e você?" - ], - [ - "Prazer em conhecê-la.", - "Obrigado." - ], - [ - "Como vai?", - "Eu estou bem." - ], - [ - "Como vai?", - "Eu estou bem. Como você está?" - ], - [ - "Oi, prazer em conhece-lo.", - "Obrigado. Prazer em conhece-lo também." - ], - [ - "É um prazer te conhecer.", - "Obrigado. Você também." - ], - [ - "O melhor do dia para você!", - "Muito obrigado." - ], - [ - "O melhor do dia para você!", - "E o resto do dia para você." - ], - [ - "E aí beleza?", - "Não muito bem." - ], - [ - "E aí beleza?", - "Não muito bem." - ], - [ - "E aí beleza?", - "Não muito bem, e você?" - ], - [ - "E aí beleza?", - "Nada de mais." - ], - [ - "E aí beleza?", - "O céu esta acima, mas eu estou bem, obrigado. E você?" - ] + "cumprimentos": [ + [ + "Olá", + "Oi" + ], + [ + "Fala", + "Falae" + ], + [ + "E ai?", + "Aqui nada, e ai?" + ], + [ + "Oi", + "Olá" + ], + [ + "Saudações!", + "Olá" + ], + [ + "Olá", + "Saudações!" + ], + [ + "E ai como vai?", + "Bem." + ], + [ + "E ai como vai?", + "Bem." + ], + [ + "Tudo bem?", + "Tudo e voce?" + ], + [ + "Bem também?", + "O que manda?" + ], + [ + "E ai como vai?", + "Ok" + ], + [ + "E ai como vai?", + "Ótimo" + ], + [ + "E ai como vai?", + "Poderia estar melhor." + ], + [ + "E ai como vai?", + "Não estou tão bem." + ], + [ + "Como vai você?", + "Bem." + ], + [ + "Como vai você?", + "Muito bem, obrigado." + ], + [ + "Como vai você?", + "Bem e você?" + ], + [ + "Prazer em conhecê-la.", + "Obrigado." + ], + [ + "Como vai?", + "Eu estou bem." + ], + [ + "Como vai?", + "Eu estou bem. Como você está?" + ], + [ + "Oi, prazer em conhece-lo.", + "Obrigado. Prazer em conhece-lo também." + ], + [ + "É um prazer te conhecer.", + "Obrigado. Você também." + ], + [ + "O melhor do dia para você!", + "Muito obrigado." + ], + [ + "O melhor do dia para você!", + "E o resto do dia para você." + ], + [ + "Bom Dia", + "Bom dia ! Tudo bem?" + ], + [ + "Bom Dia", + "Bom dia!" + ], + [ + "Boa Tarde", + "Boa Tarde!" + ], + [ + "Boa Tarde", + "Boa Tarde! Tudo bem?" + ], + [ + "Boa Noite", + "Boa Noite! Tudo bem?" + ], + [ + "Boa Noite", + "Boa Noite! " + ], + [ + "E aí beleza?", + "Não muito bem." + ], + [ + "E aí beleza?", + "Não muito bem." + ], + [ + "E aí beleza?", + "Não muito bem, e você?" + ], + [ + "E aí beleza?", + "Nada de mais." + ], + [ + "E aí beleza?", + "O céu esta acima, mas eu estou bem, obrigado. E você?" ] + ] } diff --git a/chatterbot/utils/pos_tagger.py b/chatterbot/utils/pos_tagger.py index 544ec5dba..a20f9a8c3 100644 --- a/chatterbot/utils/pos_tagger.py +++ b/chatterbot/utils/pos_tagger.py @@ -13,7 +13,7 @@ def __init__(self): from nltk import download try: - find('punkt.zip') + find('tokenizers/punkt') except LookupError: download('punkt') From a96a1411ebd72194e146b4e383f9b040a816e8eb Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Mon, 29 Aug 2016 21:27:50 -0300 Subject: [PATCH 07/43] Fix arguments (+input_statement, statement_list) for most_frequent_response , and setting to lower case fuzz.ratio, sometimes the is not close enough to the right answer (my point of view). --- chatterbot/adapters/logic/base_match.py | 2 +- chatterbot/adapters/logic/closest_match.py | 4 ++-- chatterbot/adapters/logic/mixins.py | 1 - tests/logic_adapter_tests/test_mixins.py | 1 - 4 files changed, 3 insertions(+), 5 deletions(-) diff --git a/chatterbot/adapters/logic/base_match.py b/chatterbot/adapters/logic/base_match.py index a864a7eaf..e86ae5fa9 100644 --- a/chatterbot/adapters/logic/base_match.py +++ b/chatterbot/adapters/logic/base_match.py @@ -61,7 +61,7 @@ def process(self, input_statement): len(response_list) ) ) - response = self.break_tie(input_statement,response_list, self.tie_breaking_method) + response = self.break_tie(input_statement, response_list, self.tie_breaking_method) self.logger.info(u'Tie broken. Using "{}"'.format(response.text)) else: response = self.context.storage.get_random() diff --git a/chatterbot/adapters/logic/closest_match.py b/chatterbot/adapters/logic/closest_match.py index 2dc77a322..80be46b25 100644 --- a/chatterbot/adapters/logic/closest_match.py +++ b/chatterbot/adapters/logic/closest_match.py @@ -1,7 +1,8 @@ # -*- coding: utf-8 -*- -from .base_match import BaseMatchAdapter from fuzzywuzzy import fuzz +from .base_match import BaseMatchAdapter + class ClosestMatchAdapter(BaseMatchAdapter): """ @@ -46,4 +47,3 @@ def get(self, input_statement): confidence /= 100.0 return confidence, closest_match - diff --git a/chatterbot/adapters/logic/mixins.py b/chatterbot/adapters/logic/mixins.py index e5c666017..cfeb8111d 100644 --- a/chatterbot/adapters/logic/mixins.py +++ b/chatterbot/adapters/logic/mixins.py @@ -17,7 +17,6 @@ def break_tie(self, input_statement, statement_list, method): if method in METHODS: return METHODS[method](input_statement, statement_list) - # Default to the first method if an invalid method is passed in return METHODS["first_response"](statement_list) diff --git a/tests/logic_adapter_tests/test_mixins.py b/tests/logic_adapter_tests/test_mixins.py index cad9d4869..af7fc8ce0 100644 --- a/tests/logic_adapter_tests/test_mixins.py +++ b/tests/logic_adapter_tests/test_mixins.py @@ -40,7 +40,6 @@ def test_get_random_response(self): Statement("A what?"), Statement("A phone.") ] - output = self.mixin.get_random_response(Statement("Hello"), statement_list) self.assertTrue(output) From b3cf09cc2ea6dbf9df9d8de66c8b42a0e979bd7a Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Wed, 31 Aug 2016 00:24:49 -0300 Subject: [PATCH 08/43] Initial Version, not working yet --- chatterbot/adapters/storage/sqlalchemy.py | 141 +++++++ requirements.txt | 1 + .../test_sqlalchemy_adapter.py | 361 ++++++++++++++++++ 3 files changed, 503 insertions(+) create mode 100644 chatterbot/adapters/storage/sqlalchemy.py create mode 100644 tests/storage_adapter_tests/test_sqlalchemy_adapter.py diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py new file mode 100644 index 000000000..d63b5af39 --- /dev/null +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -0,0 +1,141 @@ +from sqlalchemy import Table, MetaData, Column, Integer, ForeignKey +from sqlalchemy import Text +from sqlalchemy import create_engine +from sqlalchemy.orm import mapper + +from chatterbot.adapters.storage import StorageAdapter +from chatterbot.conversation import Response +from chatterbot.conversation import Statement + + +class SQLAlchemyDatabaseAdapter(StorageAdapter): + def __init__(self, **kwargs): + super(SQLAlchemyDatabaseAdapter, self).__init__(**kwargs) + + self.database_name = self.kwargs.get( + "database", "chatterbot-database" + ) + + # if some annoing blank space wrong... + db_name = self.database_name.strip() + + # default uses sqlite + self.database_uri = self.kwargs.get( + "database_uri", "sqlite:///" + db_name + ".db" + ) + + self.engine = create_engine(self.database_uri) + + metadata = MetaData(self.engine) + + self.response = Table('response', metadata, + Column('id', Integer, primary_key=True), + Column('text', Text), + Column('occurrence', Text), + ) + + mapper(Response, self.response) + + self.statement = Table('statement', metadata, + Column('id', Integer, primary_key=True), + Column('response_id', Integer, ForeignKey('response.id')), + Column('text', Text), + Column('extra_data', Text) + ) + + # mapper(Statement, self.statement, properties={ + # 'statement': relationship(Response, backref='response', order_by=self.statement.c.id) + # }) + + mapper(Statement, self.statement) + + self.statement.create() + self.response.create() + + def count(self): + """ + Return the number of entries in the database. + """ + raise self.AdapterMethodNotImplementedError() + + def find(self, statement_text): + """ + Returns a object from the database if it exists + """ + raise self.AdapterMethodNotImplementedError() + + def remove(self, statement_text): + """ + Removes the statement that matches the input text. + Removes any responses from statements where the response text matches + the input text. + """ + raise self.AdapterMethodNotImplementedError() + + def filter(self, **kwargs): + """ + Returns a list of objects from the database. + The kwargs parameter can contain any number + of attributes. Only objects which contain + all listed attributes and in which all values + match for all listed attributes will be returned. + """ + raise self.AdapterMethodNotImplementedError() + + def update(self, statement): + """ + Modifies an entry in the database. + Creates an entry if one does not exist. + """ + self.engine.connect() + self.statement.select().execute() + + raise self.AdapterMethodNotImplementedError() + + def get_random(self): + """ + Returns a random statement from the database + """ + raise self.AdapterMethodNotImplementedError() + + def drop(self): + """ + Drop the database attached to a given adapter. + """ + raise self.AdapterMethodNotImplementedError() + + +# Base = declarative_base() +# +# +# class ResponseTable(Base): +# __tablename__ = 'Response' +# +# id = Column(Integer, primary_key=True) +# text = Column(String) +# occurrence = Column(String) +# statement = Column(ForeignKey('Statement.id')) +# +# def __init__(self, response): +# self.text = response.text +# self.occurrence = response.occurrence +# +# def get_response(self): +# return Response(self.text, self.occurrence) +# +# +# class StatementTable(Base): +# __tablename__ = 'Statement' +# +# id = Column(Integer, primary_key=True) +# text = Column(String) +# extra_data = Column(String) +# in_response_to = relationship("Response", backref="response", order_by="Response.id") +# +# def get_statement(self): +# return Statement(self.text, self.in_response_to, self.extra_data) +# +# def __init__(self, statement): +# self.text = statement.text +# self.in_response_to = statement.in_response_to +# self.extra_data = statement.extra_data diff --git a/requirements.txt b/requirements.txt index 50c13491c..8a3361333 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,3 +4,4 @@ nltk<4.0.0 pymongo>=3.3.0,<4.0.0 python-twitter>=3.0 textblob>=0.11.0,<0.12.0 +SQLAlchemy>=1.0.14 diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py new file mode 100644 index 000000000..c0e539952 --- /dev/null +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -0,0 +1,361 @@ +from unittest import TestCase + +from chatterbot.adapters.storage.sqlalchemy import SQLAlchemyDatabaseAdapter +from chatterbot.conversation import Response +from chatterbot.conversation import Statement + + +class SQLAlchemyAdapterTestCase(TestCase): + def setUp(self): + self.adapter = SQLAlchemyDatabaseAdapter() + + +class SQLAlchemyDatabaseAdapterTestCase(SQLAlchemyAdapterTestCase): + def test_count_returns_zero(self): + """ + The count method should return a value of 0 + when nothing has been saved to the database. + """ + self.assertEqual(self.adapter.count(), 0) + + def test_count_returns_value(self): + """ + The count method should return a value of 1 + when one item has been saved to the database. + """ + statement = Statement("Test statement") + self.adapter.update(statement) + self.assertEqual(self.adapter.count(), 1) + + def test_statement_not_found(self): + """ + Test that None is returned by the find method + when a matching statement is not found. + """ + self.assertEqual(self.adapter.find("Non-existant"), None) + + def test_statement_found(self): + """ + Test that a matching statement is returned + when it exists in the database. + """ + statement = Statement("New statement") + self.adapter.update(statement) + + found_statement = self.adapter.find("New statement") + self.assertNotEqual(found_statement, None) + self.assertEqual(found_statement.text, statement.text) + + def test_update_adds_new_statement(self): + statement = Statement("New statement") + self.adapter.update(statement) + + statement_found = self.adapter.find("New statement") + self.assertNotEqual(statement_found, None) + self.assertEqual(statement_found.text, statement.text) + + def test_update_modifies_existing_statement(self): + statement = Statement("New statement") + self.adapter.update(statement) + + # Check the initial values + found_statement = self.adapter.find(statement.text) + self.assertEqual( + len(found_statement.in_response_to), 0 + ) + + # Update the statement value + statement.add_response( + Response("New response") + ) + self.adapter.update(statement) + + # Check that the values have changed + found_statement = self.adapter.find(statement.text) + self.assertEqual( + len(found_statement.in_response_to), 1 + ) + + def test_get_random_returns_statement(self): + statement = Statement("New statement") + self.adapter.update(statement) + + random_statement = self.adapter.get_random() + self.assertEqual(random_statement.text, statement.text) + + def test_find_returns_nested_responses(self): + response_list = [ + Response("Yes"), + Response("No") + ] + statement = Statement( + "Do you like this?", + in_response_to=response_list + ) + self.adapter.update(statement) + + result = self.adapter.find(statement.text) + + self.assertIn("Yes", result.in_response_to) + self.assertIn("No", result.in_response_to) + + def test_multiple_responses_added_on_update(self): + statement = Statement( + "You are welcome.", + in_response_to=[ + Response("Thank you."), + Response("Thanks.") + ] + ) + self.adapter.update(statement) + result = self.adapter.find(statement.text) + + self.assertEqual(len(result.in_response_to), 2) + self.assertIn(statement.in_response_to[0], result.in_response_to) + self.assertIn(statement.in_response_to[1], result.in_response_to) + + def test_update_saves_statement_with_multiple_responses(self): + statement = Statement( + "You are welcome.", + in_response_to=[ + Response("Thank you."), + Response("Thanks."), + ] + ) + self.adapter.update(statement) + response = self.adapter.find(statement.text) + + self.assertEqual(len(response.in_response_to), 2) + + def test_getting_and_updating_statement(self): + statement = Statement("Hi") + self.adapter.update(statement) + + statement.add_response(Response("Hello")) + statement.add_response(Response("Hello")) + self.adapter.update(statement) + + response = self.adapter.find(statement.text) + + self.assertEqual(len(response.in_response_to), 1) + self.assertEqual(response.in_response_to[0].occurrence, 2) + + def test_deserialize_responses(self): + response_list = [ + {"text": "Test", "occurrence": 3}, + {"text": "Testing", "occurrence": 1}, + ] + results = self.adapter.deserialize_responses(response_list) + + self.assertEqual(len(results), 2) + + def test_remove(self): + text = "Sometimes you have to run before you can walk." + statement = Statement(text) + self.adapter.update(statement) + self.adapter.remove(statement.text) + result = self.adapter.find(text) + + self.assertIsNone(result) + + def test_remove_response(self): + text = "Sometimes you have to run before you can walk." + statement = Statement( + "A test flight is not recommended at this design phase.", + in_response_to=[Response(text)] + ) + self.adapter.update(statement) + self.adapter.remove(statement.text) + results = self.adapter.filter(in_response_to__contains=text) + + self.assertEqual(results, []) + + def test_get_response_statements(self): + """ + Test that we are able to get a list of only statements + that are known to be in response to another statement. + """ + statement_list = [ + Statement("What... is your quest?"), + Statement("This is a phone."), + Statement("A what?", in_response_to=[Response("This is a phone.")]), + Statement("A phone.", in_response_to=[Response("A what?")]) + ] + + for statement in statement_list: + self.adapter.update(statement) + + responses = self.adapter.get_response_statements() + + self.assertEqual(len(responses), 2) + self.assertIn("This is a phone.", responses) + self.assertIn("A what?", responses) + + +class SQLAlchemyStorageAdapterFilterTestCase(SQLAlchemyAdapterTestCase): + def setUp(self): + super(SQLAlchemyAdapterTestCase, self).setUp() + + self.statement1 = Statement( + "Testing...", + in_response_to=[ + Response("Why are you counting?") + ] + ) + self.statement2 = Statement( + "Testing one, two, three.", + in_response_to=[ + Response("Testing...") + ] + ) + + def test_filter_text_no_matches(self): + self.adapter.update(self.statement1) + results = self.adapter.filter(text="Howdy") + + self.assertEqual(len(results), 0) + + def test_filter_in_response_to_no_matches(self): + self.adapter.update(self.statement1) + + results = self.adapter.filter( + in_response_to=[Response("Maybe")] + ) + self.assertEqual(len(results), 0) + + def test_filter_equal_results(self): + statement1 = Statement( + "Testing...", + in_response_to=[] + ) + statement2 = Statement( + "Testing one, two, three.", + in_response_to=[] + ) + self.adapter.update(statement1) + self.adapter.update(statement2) + + results = self.adapter.filter(in_response_to=[]) + self.assertEqual(len(results), 2) + self.assertIn(statement1, results) + self.assertIn(statement2, results) + + def test_filter_contains_result(self): + self.adapter.update(self.statement1) + self.adapter.update(self.statement2) + + results = self.adapter.filter( + in_response_to__contains="Why are you counting?" + ) + self.assertEqual(len(results), 1) + self.assertIn(self.statement1, results) + + def test_filter_contains_no_result(self): + self.adapter.update(self.statement1) + + results = self.adapter.filter( + in_response_to__contains="How do you do?" + ) + self.assertEqual(results, []) + + def test_filter_multiple_parameters(self): + self.adapter.update(self.statement1) + self.adapter.update(self.statement2) + + results = self.adapter.filter( + text="Testing...", + in_response_to__contains="Why are you counting?" + ) + + self.assertEqual(len(results), 1) + self.assertIn(self.statement1, results) + + def test_filter_multiple_parameters_no_results(self): + self.adapter.update(self.statement1) + self.adapter.update(self.statement2) + + results = self.adapter.filter( + text="Test", + in_response_to__contains="Not an existing response." + ) + + self.assertEqual(len(results), 0) + + def test_filter_no_parameters(self): + """ + If no parameters are passed to the filter, + then all statements should be returned. + """ + statement1 = Statement("Testing...") + statement2 = Statement("Testing one, two, three.") + self.adapter.update(statement1) + self.adapter.update(statement2) + + results = self.adapter.filter() + + self.assertEqual(len(results), 2) + + def test_filter_returns_statement_with_multiple_responses(self): + statement = Statement( + "You are welcome.", + in_response_to=[ + Response("Thanks."), + Response("Thank you.") + ] + ) + self.adapter.update(statement) + response = self.adapter.filter( + in_response_to__contains="Thanks." + ) + + # Get the first response + response = response[0] + + self.assertEqual(len(response.in_response_to), 2) + + def test_response_list_in_results(self): + """ + If a statement with response values is found using + the filter method, they should be returned as + response objects. + """ + statement = Statement( + "The first is to help yourself, the second is to help others.", + in_response_to=[ + Response("Why do people have two hands?") + ] + ) + self.adapter.update(statement) + found = self.adapter.filter(text=statement.text) + + self.assertEqual(len(found[0].in_response_to), 1) + self.assertEqual(type(found[0].in_response_to[0]), Response) + + +class ReadOnlySQLAlchemyDataabaseAdapterTestCase(SQLAlchemyAdapterTestCase): + def test_update_does_not_add_new_statement(self): + self.adapter.read_only = True + + statement = Statement("New statement") + self.adapter.update(statement) + + statement_found = self.adapter.find("New statement") + self.assertEqual(statement_found, None) + + def test_update_does_not_modify_existing_statement(self): + statement = Statement("New statement") + self.adapter.update(statement) + + self.adapter.read_only = True + + statement.add_response( + Response("New response") + ) + + self.adapter.update(statement) + + statement_found = self.adapter.find("New statement") + self.assertEqual(statement_found.text, statement.text) + self.assertEqual( + len(statement_found.in_response_to), 0 + ) From 6b1a616e9d631c5b28941220f9d1c60ad4871602 Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Wed, 31 Aug 2016 12:05:14 -0300 Subject: [PATCH 09/43] Some work done, still need to implement other methods. --- chatterbot/adapters/storage/sqlalchemy.py | 190 ++++++++++++++-------- 1 file changed, 120 insertions(+), 70 deletions(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index d63b5af39..7e270b464 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -1,12 +1,58 @@ -from sqlalchemy import Table, MetaData, Column, Integer, ForeignKey -from sqlalchemy import Text +import json + +from sqlalchemy import Column, ForeignKey +from sqlalchemy import String from sqlalchemy import create_engine -from sqlalchemy.orm import mapper +from sqlalchemy.ext.declarative import declarative_base +from sqlalchemy.orm import relationship +from sqlalchemy.orm import sessionmaker from chatterbot.adapters.storage import StorageAdapter from chatterbot.conversation import Response from chatterbot.conversation import Statement +Base = declarative_base() + + +class StatementTable(Base): + __tablename__ = 'StatementTable' + + # id = Column(Integer) + text = Column(String, primary_key=True) + extra_data = Column(String) + in_response_to = relationship("ResponseTable", back_populates="statementTable") + + # in_response_to = relationship("Response", backref="response", order_by="Response.id") + def get_statement(self): + return Statement(self.text, self.in_response_to.text, + self.extra_data) + + # def __init__(self, text, extra_data, in_response_to): + # std = StatementTable + # std.text = st1.text + # std.extra_data = st1.extra_data + # # if statement.in_response_to: + # # self.in_response_to = statement.in_response_to + # # self.extra_data = + # return std + + +class ResponseTable(Base): + __tablename__ = 'ResponseTable' + + # id = Column(Integer) + text = Column(String, primary_key=True) + occurrence = Column(String) + statement_text = Column(String, ForeignKey('StatementTable.text')) + statementTable = relationship("StatementTable", back_populates="in_response_to") + + # def __init__(self, response): + # self.text = response.text + # self.occurrence = response.occurrence + + def get_response(self): + return Response(self.text, self.occurrence) + class SQLAlchemyDatabaseAdapter(StorageAdapter): def __init__(self, **kwargs): @@ -26,43 +72,60 @@ def __init__(self, **kwargs): self.engine = create_engine(self.database_uri) - metadata = MetaData(self.engine) - - self.response = Table('response', metadata, - Column('id', Integer, primary_key=True), - Column('text', Text), - Column('occurrence', Text), - ) - - mapper(Response, self.response) - - self.statement = Table('statement', metadata, - Column('id', Integer, primary_key=True), - Column('response_id', Integer, ForeignKey('response.id')), - Column('text', Text), - Column('extra_data', Text) - ) - - # mapper(Statement, self.statement, properties={ - # 'statement': relationship(Response, backref='response', order_by=self.statement.c.id) - # }) - - mapper(Statement, self.statement) - - self.statement.create() - self.response.create() + # metadata = MetaData(self.engine) + + # Recreate database + # metadata.drop_all() + # + # self.response = Table('response', metadata, + # Column('id', Integer, primary_key=True), + # Column('text', Text), + # Column('occurrence', Text), + # ) + # + # self.statement = Table('statement', metadata, + # Column('id', Integer, primary_key=True), + # Column('text', Text), + # Column('in_response_to', Integer, ForeignKey('response.id')), + # Column('extra_data', Text) + # ) + # + # # mapper(Response, self.response, + # # # non_primary=True, + # # properties={ + # # 'statement': relationship(Statement, backref='response') + # # }, ) + # mapper(Statement, self.statement) + # mapper(Response, self.response) + + Base.metadata.drop_all(self.engine) + Base.metadata.create_all(self.engine) def count(self): """ Return the number of entries in the database. """ - raise self.AdapterMethodNotImplementedError() + Session = sessionmaker(bind=self.engine) + session = Session() + + return session.query(StatementTable).count() def find(self, statement_text): """ Returns a object from the database if it exists """ - raise self.AdapterMethodNotImplementedError() + Session = sessionmaker(bind=self.engine) + session = Session() + + std = session.query(StatementTable).filter_by(text=statement_text).first() + # extra_data = json.loads(std.extra_data) + # if extra_data: + # extra_data = dict[extra_data] + # TODO Extra data + if std: + return Statement(std.text) + else: + return None def remove(self, statement_text): """ @@ -70,7 +133,15 @@ def remove(self, statement_text): Removes any responses from statements where the response text matches the input text. """ - raise self.AdapterMethodNotImplementedError() + + Session = sessionmaker(bind=self.engine) + session = Session() + + std = session.query(StatementTable).filter_by(text=statement_text).firs() + session.delete(std) + session.commit() + + # raise self.AdapterMethodNotImplementedError() def filter(self, **kwargs): """ @@ -80,6 +151,7 @@ def filter(self, **kwargs): all listed attributes and in which all values match for all listed attributes will be returned. """ + raise self.AdapterMethodNotImplementedError() def update(self, statement): @@ -87,10 +159,24 @@ def update(self, statement): Modifies an entry in the database. Creates an entry if one does not exist. """ - self.engine.connect() - self.statement.select().execute() - raise self.AdapterMethodNotImplementedError() + if statement: + Session = sessionmaker(bind=self.engine) + session = Session() + std = session.query(StatementTable).filter_by(text=statement.text).first() + if std: + # update + if statement.text: + std.text = statement.text + if statement.extra_data: + std.extra_data = json.dumps(statement.extra_data) + session.add(std) + else: + session.add(StatementTable(text=statement.text)) + + session.commit() + + # raise self.AdapterMethodNotImplementedError() def get_random(self): """ @@ -103,39 +189,3 @@ def drop(self): Drop the database attached to a given adapter. """ raise self.AdapterMethodNotImplementedError() - - -# Base = declarative_base() -# -# -# class ResponseTable(Base): -# __tablename__ = 'Response' -# -# id = Column(Integer, primary_key=True) -# text = Column(String) -# occurrence = Column(String) -# statement = Column(ForeignKey('Statement.id')) -# -# def __init__(self, response): -# self.text = response.text -# self.occurrence = response.occurrence -# -# def get_response(self): -# return Response(self.text, self.occurrence) -# -# -# class StatementTable(Base): -# __tablename__ = 'Statement' -# -# id = Column(Integer, primary_key=True) -# text = Column(String) -# extra_data = Column(String) -# in_response_to = relationship("Response", backref="response", order_by="Response.id") -# -# def get_statement(self): -# return Statement(self.text, self.in_response_to, self.extra_data) -# -# def __init__(self, statement): -# self.text = statement.text -# self.in_response_to = statement.in_response_to -# self.extra_data = statement.extra_data From cca2882bedbb648f8af9bdc0dcebb8abe369f42e Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Thu, 1 Sep 2016 13:29:17 +0530 Subject: [PATCH 10/43] minor typ --- chatterbot/adapters/storage/sqlalchemy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index 7e270b464..3bc89157d 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -137,7 +137,7 @@ def remove(self, statement_text): Session = sessionmaker(bind=self.engine) session = Session() - std = session.query(StatementTable).filter_by(text=statement_text).firs() + std = session.query(StatementTable).filter_by(text=statement_text).first() session.delete(std) session.commit() From bdec3dded55ce369e28d783e03be34f0e5593ee8 Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Thu, 1 Sep 2016 14:04:28 +0530 Subject: [PATCH 11/43] assert method for None checks --- tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index c0e539952..be8df2503 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -32,7 +32,7 @@ def test_statement_not_found(self): Test that None is returned by the find method when a matching statement is not found. """ - self.assertEqual(self.adapter.find("Non-existant"), None) + self.assertIsNone(self.adapter.find("Non-existant")) def test_statement_found(self): """ @@ -43,7 +43,7 @@ def test_statement_found(self): self.adapter.update(statement) found_statement = self.adapter.find("New statement") - self.assertNotEqual(found_statement, None) + self.assertIsNotNone(found_statement) self.assertEqual(found_statement.text, statement.text) def test_update_adds_new_statement(self): @@ -51,7 +51,7 @@ def test_update_adds_new_statement(self): self.adapter.update(statement) statement_found = self.adapter.find("New statement") - self.assertNotEqual(statement_found, None) + self.assertIsNotNone(statement_found) self.assertEqual(statement_found.text, statement.text) def test_update_modifies_existing_statement(self): @@ -340,7 +340,7 @@ def test_update_does_not_add_new_statement(self): self.adapter.update(statement) statement_found = self.adapter.find("New statement") - self.assertEqual(statement_found, None) + self.assertIsNone(statement_found) def test_update_does_not_modify_existing_statement(self): statement = Statement("New statement") From 427b42a9578b0b656f3f1ef14c9b427a6ca5af71 Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Thu, 1 Sep 2016 19:11:29 +0530 Subject: [PATCH 12/43] super argument fix --- tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index be8df2503..d224ef4bc 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -194,7 +194,7 @@ def test_get_response_statements(self): class SQLAlchemyStorageAdapterFilterTestCase(SQLAlchemyAdapterTestCase): def setUp(self): - super(SQLAlchemyAdapterTestCase, self).setUp() + super(SQLAlchemyStorageAdapterFilterTestCase, self).setUp() self.statement1 = Statement( "Testing...", From a7999cbfc592c1dbb8cda1dfebc8ca217c3b9c2c Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Thu, 1 Sep 2016 19:21:03 +0530 Subject: [PATCH 13/43] assertIsInstance for type check --- tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index d224ef4bc..7ee223e98 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -329,7 +329,7 @@ def test_response_list_in_results(self): found = self.adapter.filter(text=statement.text) self.assertEqual(len(found[0].in_response_to), 1) - self.assertEqual(type(found[0].in_response_to[0]), Response) + self.assertIsInstance(found[0].in_response_to[0], Response) class ReadOnlySQLAlchemyDataabaseAdapterTestCase(SQLAlchemyAdapterTestCase): From 2ea136b5c93448df555cb60bf95668de3cf21631 Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Fri, 2 Sep 2016 00:30:55 -0300 Subject: [PATCH 14/43] Some work done, still need to implement filter. --- chatterbot/adapters/storage/sqlalchemy.py | 135 ++++++++++++------ .../test_sqlalchemy_adapter.py | 11 +- 2 files changed, 92 insertions(+), 54 deletions(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index 3bc89157d..68d4a40b7 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -1,10 +1,12 @@ import json +import random from sqlalchemy import Column, ForeignKey +from sqlalchemy import PickleType from sqlalchemy import String from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship +from sqlalchemy.orm import relationship, backref from sqlalchemy.orm import sessionmaker from chatterbot.adapters.storage import StorageAdapter @@ -14,44 +16,62 @@ Base = declarative_base() +def get_statement_table_data(context): + print(context.get_statement) + pass + + class StatementTable(Base): __tablename__ = 'StatementTable' - # id = Column(Integer) - text = Column(String, primary_key=True) - extra_data = Column(String) - in_response_to = relationship("ResponseTable", back_populates="statementTable") - - # in_response_to = relationship("Response", backref="response", order_by="Response.id") def get_statement(self): - return Statement(self.text, self.in_response_to.text, - self.extra_data) + stmt = Statement(self.text, **self.extra_data) + for resp in self.in_response_to: + stmt.add_response(resp.get_response()) + return stmt - # def __init__(self, text, extra_data, in_response_to): - # std = StatementTable - # std.text = st1.text - # std.extra_data = st1.extra_data - # # if statement.in_response_to: - # # self.in_response_to = statement.in_response_to - # # self.extra_data = - # return std + def get_statement_serialized(context): + params = context.current_parameters + del (params['text_search']) + return json.dumps(params) + + # id = Column(Integer) + text = Column(String, primary_key=True) + extra_data = Column(PickleType) + # in_response_to = relationship("ResponseTable", back_populates="statement_table") + text_search = Column(String, primary_key=True, default=get_statement_serialized) class ResponseTable(Base): __tablename__ = 'ResponseTable' + def get_reponse_serialized(context): + params = context.current_parameters + del (params['text_search']) + return json.dumps(params) + # id = Column(Integer) text = Column(String, primary_key=True) occurrence = Column(String) statement_text = Column(String, ForeignKey('StatementTable.text')) - statementTable = relationship("StatementTable", back_populates="in_response_to") - - # def __init__(self, response): - # self.text = response.text - # self.occurrence = response.occurrence + statement_table = relationship("StatementTable", backref=backref('in_response_to'), cascade="all, delete-orphan", + single_parent=True) + text_search = Column(String, primary_key=True, default=get_reponse_serialized) def get_response(self): - return Response(self.text, self.occurrence) + occ = {"occurrence": self.occurrence} + return Response(text=self.text, **occ) + + +def get_statement_table(statement): + responses = [] + for resp in statement.in_response_to: + responses.append(get_response_table(resp)) + return StatementTable(text=statement.text, in_response_to=responses, extra_data=statement.extra_data) + + +def get_response_table(response): + return ResponseTable(text=response.text, occurrence=response.occurrence) class SQLAlchemyDatabaseAdapter(StorageAdapter): @@ -105,25 +125,28 @@ def count(self): """ Return the number of entries in the database. """ + session = self.get_session() + return session.query(StatementTable).count() + + def get_session(self): + """ + :rtype: Session + """ Session = sessionmaker(bind=self.engine) session = Session() - return session.query(StatementTable).count() + return session def find(self, statement_text): """ Returns a object from the database if it exists """ - Session = sessionmaker(bind=self.engine) - session = Session() + session = self.get_session() std = session.query(StatementTable).filter_by(text=statement_text).first() - # extra_data = json.loads(std.extra_data) - # if extra_data: - # extra_data = dict[extra_data] - # TODO Extra data + if std: - return Statement(std.text) + return std.get_statement() else: return None @@ -134,14 +157,16 @@ def remove(self, statement_text): the input text. """ - Session = sessionmaker(bind=self.engine) - session = Session() + session = self.get_session() std = session.query(StatementTable).filter_by(text=statement_text).first() session.delete(std) - session.commit() - # raise self.AdapterMethodNotImplementedError() + if not self.read_only: + session.commit() + else: + session.rollback() + # raise self.AdapterMethodNotImplementedError() def filter(self, **kwargs): """ @@ -152,7 +177,21 @@ def filter(self, **kwargs): match for all listed attributes will be returned. """ - raise self.AdapterMethodNotImplementedError() + filter_parameters = kwargs.copy() + + session = self.get_session() + session.query() + stmts = [] + for fp in filter_parameters: + stmts.extend(session.query(ResponseTable).filter( + ResponseTable.text_search.like('%' + filter_parameters[fp] + '%')).all()) + + results = [] + for st in stmts: + if st and st.statement_table: + results.append(st.statement_table.get_statement()) + + return results def update(self, statement): """ @@ -161,31 +200,39 @@ def update(self, statement): """ if statement: - Session = sessionmaker(bind=self.engine) - session = Session() + session = self.get_session() std = session.query(StatementTable).filter_by(text=statement.text).first() if std: # update if statement.text: std.text = statement.text if statement.extra_data: - std.extra_data = json.dumps(statement.extra_data) + std.extra_data = dict[statement.extra_data] session.add(std) else: - session.add(StatementTable(text=statement.text)) + session.add(get_statement_table(statement)) + if not self.read_only: session.commit() - - # raise self.AdapterMethodNotImplementedError() + else: + session.rollback() def get_random(self): """ Returns a random statement from the database """ - raise self.AdapterMethodNotImplementedError() + count = self.count() + if count < 1: + raise self.EmptyDatabaseException() + + rand = random.randrange(0, count) + session = self.get_session() + stmt = session.query(StatementTable)[rand] + + return stmt.get_statement() def drop(self): """ Drop the database attached to a given adapter. """ - raise self.AdapterMethodNotImplementedError() + Base.metadata.drop_all(self.engine) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index 7ee223e98..f2455d817 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -140,15 +140,6 @@ def test_getting_and_updating_statement(self): self.assertEqual(len(response.in_response_to), 1) self.assertEqual(response.in_response_to[0].occurrence, 2) - def test_deserialize_responses(self): - response_list = [ - {"text": "Test", "occurrence": 3}, - {"text": "Testing", "occurrence": 1}, - ] - results = self.adapter.deserialize_responses(response_list) - - self.assertEqual(len(results), 2) - def test_remove(self): text = "Sometimes you have to run before you can walk." statement = Statement(text) @@ -332,7 +323,7 @@ def test_response_list_in_results(self): self.assertIsInstance(found[0].in_response_to[0], Response) -class ReadOnlySQLAlchemyDataabaseAdapterTestCase(SQLAlchemyAdapterTestCase): +class ReadOnlySQLAlchemyDatabaseAdapterTestCase(SQLAlchemyAdapterTestCase): def test_update_does_not_add_new_statement(self): self.adapter.read_only = True From 2f19f76b218baa509f63b483870ef29d161c8ce7 Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Fri, 2 Sep 2016 10:31:45 +0530 Subject: [PATCH 15/43] get_session is private --- chatterbot-database.db | Bin 0 -> 5120 bytes chatterbot/adapters/storage/sqlalchemy.py | 15 +++++++-------- 2 files changed, 7 insertions(+), 8 deletions(-) create mode 100644 chatterbot-database.db diff --git a/chatterbot-database.db b/chatterbot-database.db new file mode 100644 index 0000000000000000000000000000000000000000..d69204d2def68e2a7e6d26a2f58f16252a47dcfe GIT binary patch literal 5120 zcmeHKL5tcz6rM3rtFUyT2QPbg1Qt!Pg7i?@9@;2byVO-1ZOfiQR%eNziJKYQT|{W? zUugfw9t*wq*#FX7kDalEbtpgv^`w=Dm4e=FQ7=Up0q}z?M0<fpe zLy(Ip>4(8!RUY!;h*m!6pGA%ent!UyjGF_*cmlI|84Pjs@xHN*7h|614zPBHcvGYh zPzdZCf%t%*2C97jZ8b&}Y@{viatU#VdGqrI>GYYH&rG+<=zXuwB*v4$?Ze{KKp86#_dzAg-hv Pi^JT;Jf0c Date: Fri, 2 Sep 2016 11:45:38 +0530 Subject: [PATCH 16/43] wrapper for filter --- chatterbot-database.db | Bin 5120 -> 0 bytes chatterbot/adapters/storage/sqlalchemy.py | 44 +++++++++++++--------- 2 files changed, 26 insertions(+), 18 deletions(-) delete mode 100644 chatterbot-database.db diff --git a/chatterbot-database.db b/chatterbot-database.db deleted file mode 100644 index d69204d2def68e2a7e6d26a2f58f16252a47dcfe..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 5120 zcmeHKL5tcz6rM3rtFUyT2QPbg1Qt!Pg7i?@9@;2byVO-1ZOfiQR%eNziJKYQT|{W? zUugfw9t*wq*#FX7kDalEbtpgv^`w=Dm4e=FQ7=Up0q}z?M0<fpe zLy(Ip>4(8!RUY!;h*m!6pGA%ent!UyjGF_*cmlI|84Pjs@xHN*7h|614zPBHcvGYh zPzdZCf%t%*2C97jZ8b&}Y@{viatU#VdGqrI>GYYH&rG+<=zXuwB*v4$?Ze{KKp86#_dzAg-hv Pi^JT;Jf0c Date: Fri, 2 Sep 2016 12:21:34 +0530 Subject: [PATCH 17/43] looping via map --- chatterbot/adapters/storage/sqlalchemy.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index c8631d671..2158b2831 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -62,11 +62,11 @@ def get_response(self): occ = {"occurrence": self.occurrence} return Response(text=self.text, **occ) +def poi(s): + return s*s def get_statement_table(statement): - responses = [] - for resp in statement.in_response_to: - responses.append(get_response_table(resp)) + responses = list(map(get_response_table, statement.in_response_to)) return StatementTable(text=statement.text, in_response_to=responses, extra_data=statement.extra_data) From d05541a809672bfdbfc4b9bceecb6a3f5e64003b Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Fri, 2 Sep 2016 12:24:24 +0530 Subject: [PATCH 18/43] poi removed --- chatterbot/adapters/storage/sqlalchemy.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index 2158b2831..db8c4ddda 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -62,8 +62,6 @@ def get_response(self): occ = {"occurrence": self.occurrence} return Response(text=self.text, **occ) -def poi(s): - return s*s def get_statement_table(statement): responses = list(map(get_response_table, statement.in_response_to)) From 97b09a050ff18d564880e1f854f1237349fbe117 Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Fri, 2 Sep 2016 21:52:16 +0530 Subject: [PATCH 19/43] minor refactor --- chatterbot/adapters/storage/sqlalchemy.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index db8c4ddda..140feec61 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -183,11 +183,12 @@ def filter(self, **kwargs): filter_parameters = kwargs.copy() session = self.__get_session() - session.query() stmts = [] for fp in filter_parameters: - stmts.extend(session.query(ResponseTable).filter( - ResponseTable.text_search.like('%' + filter_parameters[fp] + '%')).all()) + _like = filter_parameters[fp] + _response_query = session.query(ResponseTable) + query = _response_query.filter(ResponseTable.text_search.like('%' + _like + '%')) + stmts.extend(query.all()) results = [] for st in stmts: From 3fa6ac7da8df801d938a428736f7ac86cf8502f2 Mon Sep 17 00:00:00 2001 From: naveen yadav Date: Fri, 2 Sep 2016 21:57:57 +0530 Subject: [PATCH 20/43] informatve variable name --- chatterbot/adapters/storage/sqlalchemy.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index 140feec61..8a4c371f5 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -183,17 +183,17 @@ def filter(self, **kwargs): filter_parameters = kwargs.copy() session = self.__get_session() - stmts = [] + statements = [] for fp in filter_parameters: _like = filter_parameters[fp] _response_query = session.query(ResponseTable) query = _response_query.filter(ResponseTable.text_search.like('%' + _like + '%')) - stmts.extend(query.all()) + statements.extend(query.all()) results = [] - for st in stmts: - if st and st.statement_table: - results.append(st.statement_table.get_statement()) + for statement in statements: + if statement and statement.statement_table: + results.append(statement.statement_table.get_statement()) return results From d87b2d812a38f998979db31cff6b6075b7d75b9e Mon Sep 17 00:00:00 2001 From: davi_zucon Date: Mon, 5 Sep 2016 10:07:53 -0300 Subject: [PATCH 21/43] merge changes from masters --- .gitignore | 4 +- chatterbot/adapters/storage/sqlalchemy.py | 56 ++++++++++++++++------- 2 files changed, 43 insertions(+), 17 deletions(-) diff --git a/.gitignore b/.gitignore index 9f5eb8e29..5b96d9e0c 100644 --- a/.gitignore +++ b/.gitignore @@ -15,4 +15,6 @@ docs/_build/ # IntelliJ .idea -*.iml \ No newline at end of file +*.iml +*. +chatterbot-database.db diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index 8a4c371f5..b0575f6ce 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -6,7 +6,7 @@ from sqlalchemy import String from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship, backref +from sqlalchemy.orm import relationship from sqlalchemy.orm import sessionmaker from chatterbot.adapters.storage import StorageAdapter @@ -16,11 +16,6 @@ Base = declarative_base() -def get_statement_table_data(context): - print(context.get_statement) - pass - - class StatementTable(Base): __tablename__ = 'StatementTable' @@ -38,7 +33,10 @@ def get_statement_serialized(context): # id = Column(Integer) text = Column(String, primary_key=True) extra_data = Column(PickleType) - # in_response_to = relationship("ResponseTable", back_populates="statement_table") + # Old: in_response_to = relationship("ResponseTable", back_populates="statement_table") + # relationship: + in_response_to = relationship("ResponseTable", back_populates="statement_table") + text_search = Column(String, primary_key=True, default=get_statement_serialized) @@ -54,8 +52,12 @@ def get_reponse_serialized(context): text = Column(String, primary_key=True) occurrence = Column(String) statement_text = Column(String, ForeignKey('StatementTable.text')) - statement_table = relationship("StatementTable", backref=backref('in_response_to'), cascade="all, delete-orphan", - single_parent=True) + + # Old: statement_table = relationship("StatementTable", backref=backref('in_response_to'), cascade="all, delete-orphan", single_parent=True) + # Test relationship: + statement_table = relationship("StatementTable", back_populates="in_response_to", cascade="all", + uselist=False) + text_search = Column(String, primary_key=True, default=get_reponse_serialized) def get_response(self): @@ -184,16 +186,38 @@ def filter(self, **kwargs): session = self.__get_session() statements = [] - for fp in filter_parameters: - _like = filter_parameters[fp] - _response_query = session.query(ResponseTable) - query = _response_query.filter(ResponseTable.text_search.like('%' + _like + '%')) - statements.extend(query.all()) + + if len(filter_parameters) == 0: + _response_query = session.query(StatementTable) + statements.extend(_response_query.all()) + else: + for fp in filter_parameters: + _like = filter_parameters[fp] + if fp == 'in_response_to': + _response_query = session.query(StatementTable) + if isinstance(_like, list): + if len(_like) == 0: + query = _response_query.filter(StatementTable.in_response_to == None,StatementTable.subject_id != None) + else: + query = _response_query.filter(StatementTable.in_response_to.contain(_like)) + else: + query = _response_query.filter(StatementTable.in_response_to.like('%' + _like + '%')) + + + else: + _response_query = session.query(ResponseTable) + query = _response_query.filter(ResponseTable.text_search.like('%' + _like + '%')) + + statements.extend(query.all()) results = [] for statement in statements: - if statement and statement.statement_table: - results.append(statement.statement_table.get_statement()) + if isinstance(statement, ResponseTable): + if statement and statement.statement_table: + results.append(statement.statement_table.get_statement()) + else: + if statement: + results.append(statement.get_statement()) return results From 287146725ec352d93b53eafcfb4e45ccb575b69d Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Thu, 2 Mar 2017 22:46:19 -0300 Subject: [PATCH 22/43] must review this commit, long time away --- chatterbot/adapters/storage/sqlalchemy.py | 1 + tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 1 - 2 files changed, 1 insertion(+), 1 deletion(-) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/adapters/storage/sqlalchemy.py index b0575f6ce..34ab84023 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/adapters/storage/sqlalchemy.py @@ -16,6 +16,7 @@ Base = declarative_base() + class StatementTable(Base): __tablename__ = 'StatementTable' diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index f2455d817..3fffd5054 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -9,7 +9,6 @@ class SQLAlchemyAdapterTestCase(TestCase): def setUp(self): self.adapter = SQLAlchemyDatabaseAdapter() - class SQLAlchemyDatabaseAdapterTestCase(SQLAlchemyAdapterTestCase): def test_count_returns_zero(self): """ From 7c7015484a43416501b16a934d7c472025456a8f Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 29 Mar 2017 17:25:54 -0300 Subject: [PATCH 23/43] merge from master --- .gitignore | 2 ++ 1 file changed, 2 insertions(+) diff --git a/.gitignore b/.gitignore index 116ddb6e9..412e41634 100644 --- a/.gitignore +++ b/.gitignore @@ -23,3 +23,5 @@ docs/_build/ examples/settings.py examples/ubuntu_dialogs* +.env +.out From 03667223e190b4f4dce57eaff9df5a99143fe75c Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 29 Mar 2017 17:48:12 -0300 Subject: [PATCH 24/43] move to new package and merge --- chatterbot/{adapters => }/storage/sqlalchemy.py | 8 +++++--- tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 7 ++++--- 2 files changed, 9 insertions(+), 6 deletions(-) rename chatterbot/{adapters => }/storage/sqlalchemy.py (97%) diff --git a/chatterbot/adapters/storage/sqlalchemy.py b/chatterbot/storage/sqlalchemy.py similarity index 97% rename from chatterbot/adapters/storage/sqlalchemy.py rename to chatterbot/storage/sqlalchemy.py index 34ab84023..ab583a4c1 100644 --- a/chatterbot/adapters/storage/sqlalchemy.py +++ b/chatterbot/storage/sqlalchemy.py @@ -9,14 +9,13 @@ from sqlalchemy.orm import relationship from sqlalchemy.orm import sessionmaker -from chatterbot.adapters.storage import StorageAdapter +from chatterbot.storage import StorageAdapter from chatterbot.conversation import Response from chatterbot.conversation import Statement Base = declarative_base() - class StatementTable(Base): __tablename__ = 'StatementTable' @@ -157,6 +156,8 @@ def find(self, statement_text): return record.get_statement() return None + read_only = False + def remove(self, statement_text): """ Removes the statement that matches the input text. @@ -198,7 +199,8 @@ def filter(self, **kwargs): _response_query = session.query(StatementTable) if isinstance(_like, list): if len(_like) == 0: - query = _response_query.filter(StatementTable.in_response_to == None,StatementTable.subject_id != None) + query = _response_query.filter(StatementTable.in_response_to == None, + StatementTable.subject_id != None) else: query = _response_query.filter(StatementTable.in_response_to.contain(_like)) else: diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index 3fffd5054..e6001eafb 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -1,15 +1,16 @@ from unittest import TestCase -from chatterbot.adapters.storage.sqlalchemy import SQLAlchemyDatabaseAdapter -from chatterbot.conversation import Response -from chatterbot.conversation import Statement +from chatterbot.conversation import Statement, Response +from chatterbot.storage.sqlalchemy import SQLAlchemyDatabaseAdapter class SQLAlchemyAdapterTestCase(TestCase): def setUp(self): self.adapter = SQLAlchemyDatabaseAdapter() + class SQLAlchemyDatabaseAdapterTestCase(SQLAlchemyAdapterTestCase): + def test_count_returns_zero(self): """ The count method should return a value of 0 From 6702ca2ccbb7c5e36d8505193ee27bec0642e08c Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 29 Mar 2017 18:02:50 -0300 Subject: [PATCH 25/43] All testes passing # Conflicts: # requirements.txt --- chatterbot/storage/sqlalchemy.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/chatterbot/storage/sqlalchemy.py b/chatterbot/storage/sqlalchemy.py index ab583a4c1..5f48e9e03 100644 --- a/chatterbot/storage/sqlalchemy.py +++ b/chatterbot/storage/sqlalchemy.py @@ -4,6 +4,7 @@ from sqlalchemy import Column, ForeignKey from sqlalchemy import PickleType from sqlalchemy import String +from sqlalchemy import Integer from sqlalchemy import create_engine from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship @@ -50,7 +51,7 @@ def get_reponse_serialized(context): # id = Column(Integer) text = Column(String, primary_key=True) - occurrence = Column(String) + occurrence = Column(Integer) statement_text = Column(String, ForeignKey('StatementTable.text')) # Old: statement_table = relationship("StatementTable", backref=backref('in_response_to'), cascade="all, delete-orphan", single_parent=True) @@ -241,6 +242,8 @@ def update(self, statement): record.text = statement.text if statement.extra_data: record.extra_data = dict[statement.extra_data] + if statement.in_response_to: + record.in_response_to = list(map(get_response_table, statement.in_response_to)) session.add(record) else: session.add(get_statement_table(statement)) From 8e27a2de8c2f7d07b98bafbb7e58fc147f7f5c80 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Tue, 18 Apr 2017 23:51:39 -0300 Subject: [PATCH 26/43] to check... --- chatterbot/storage/__init__.py | 1 + chatterbot/storage/sqlalchemy.py | 146 ++++++++++++++---- requirements.txt | 2 +- setup.cfg | 2 +- tests/base_case.py | 12 ++ .../sqlalchemy_integration_tests.py | 17 ++ .../test_sqlalchemy_adapter.py | 3 +- tests/teste_main.py | 40 +++++ .../test_chatterbot_corpus_training.py | 1 - 9 files changed, 188 insertions(+), 36 deletions(-) create mode 100644 tests/storage_adapter_tests/integration_tests/sqlalchemy_integration_tests.py create mode 100644 tests/teste_main.py diff --git a/chatterbot/storage/__init__.py b/chatterbot/storage/__init__.py index 2fe7e08b9..658d933a7 100644 --- a/chatterbot/storage/__init__.py +++ b/chatterbot/storage/__init__.py @@ -2,3 +2,4 @@ from .django_storage import DjangoStorageAdapter from .jsonfile import JsonFileStorageAdapter from .mongodb import MongoDatabaseAdapter +from .sqlalchemy import SQLAlchemyDatabaseAdapter diff --git a/chatterbot/storage/sqlalchemy.py b/chatterbot/storage/sqlalchemy.py index 5f48e9e03..99ad7be41 100644 --- a/chatterbot/storage/sqlalchemy.py +++ b/chatterbot/storage/sqlalchemy.py @@ -1,11 +1,13 @@ import json import random +import sqlalchemy from sqlalchemy import Column, ForeignKey from sqlalchemy import PickleType from sqlalchemy import String from sqlalchemy import Integer from sqlalchemy import create_engine +from sqlalchemy.exc import DatabaseError from sqlalchemy.ext.declarative import declarative_base from sqlalchemy.orm import relationship from sqlalchemy.orm import sessionmaker @@ -31,13 +33,11 @@ def get_statement_serialized(context): del (params['text_search']) return json.dumps(params) - # id = Column(Integer) + id = Column(Integer) text = Column(String, primary_key=True) extra_data = Column(PickleType) - # Old: in_response_to = relationship("ResponseTable", back_populates="statement_table") # relationship: in_response_to = relationship("ResponseTable", back_populates="statement_table") - text_search = Column(String, primary_key=True, default=get_statement_serialized) @@ -49,16 +49,14 @@ def get_reponse_serialized(context): del (params['text_search']) return json.dumps(params) - # id = Column(Integer) + id = Column(Integer) text = Column(String, primary_key=True) occurrence = Column(Integer) statement_text = Column(String, ForeignKey('StatementTable.text')) # Old: statement_table = relationship("StatementTable", backref=backref('in_response_to'), cascade="all, delete-orphan", single_parent=True) # Test relationship: - statement_table = relationship("StatementTable", back_populates="in_response_to", cascade="all", - uselist=False) - + statement_table = relationship("StatementTable", back_populates="in_response_to", cascade="all", uselist=False) text_search = Column(String, primary_key=True, default=get_reponse_serialized) def get_response(self): @@ -66,6 +64,14 @@ def get_response(self): return Response(text=self.text, **occ) +# # relational table TO REMOVE +# in_response_to = sqlalchemy.Table('in_responses_to', +# Base.metadata, +# sqlalchemy.Column('stmt_id', sqlalchemy.Integer, ForeignKey('StatementTable.id')), +# sqlalchemy.Column('resp_id', sqlalchemy.Integer, +# sqlalchemy.ForeignKey('ResponseTable.id'))) + + def get_statement_table(statement): responses = list(map(get_response_table, statement.in_response_to)) return StatementTable(text=statement.text, in_response_to=responses, extra_data=statement.extra_data) @@ -76,6 +82,9 @@ def get_response_table(response): class SQLAlchemyDatabaseAdapter(StorageAdapter): + read_only = False + drop_create = False + def __init__(self, **kwargs): super(SQLAlchemyDatabaseAdapter, self).__init__(**kwargs) @@ -119,8 +128,17 @@ def __init__(self, **kwargs): # mapper(Statement, self.statement) # mapper(Response, self.response) - Base.metadata.drop_all(self.engine) - Base.metadata.create_all(self.engine) + self.read_only = self.kwargs.get( + "read_only", False + ) + + self.drop_create = self.kwargs.get( + "drop_create", False + ) + + if not self.read_only and self.drop_create: + Base.metadata.drop_all(self.engine) + Base.metadata.create_all(self.engine) def count(self): """ @@ -139,7 +157,7 @@ def __get_session(self): def __statement_filter(self, session, **kwargs): """ - Apply filter opeartion on StatementTable + Apply filter operation on StatementTable rtype: query """ @@ -157,8 +175,6 @@ def find(self, statement_text): return record.get_statement() return None - read_only = False - def remove(self, statement_text): """ Removes the statement that matches the input text. @@ -170,11 +186,13 @@ def remove(self, statement_text): record = query.first() session.delete(record) - if not self.read_only: - session.commit() - else: - session.rollback() - # raise self.AdapterMethodNotImplementedError() + try: + if not self.read_only: + session.commit() + else: + session.rollback() + except DatabaseError as e: + self.logger.error(statement_text, str(e.orig)) def filter(self, **kwargs): """ @@ -195,22 +213,27 @@ def filter(self, **kwargs): statements.extend(_response_query.all()) else: for fp in filter_parameters: - _like = filter_parameters[fp] - if fp == 'in_response_to': + _filter = filter_parameters[fp] + if fp in ['in_response_to', 'in_response_to__contains']: _response_query = session.query(StatementTable) - if isinstance(_like, list): - if len(_like) == 0: - query = _response_query.filter(StatementTable.in_response_to == None, - StatementTable.subject_id != None) + if isinstance(_filter, list): + if len(_filter) == 0: + query = _response_query.filter( + StatementTable.in_response_to == None) # Here must use == instead of is else: - query = _response_query.filter(StatementTable.in_response_to.contain(_like)) + for f in _filter: + query = _response_query.filter( + StatementTable.in_response_to.contains(get_response_table(f))) else: - query = _response_query.filter(StatementTable.in_response_to.like('%' + _like + '%')) - + # if fp == 'in_response_to__contains': + # FIXME Optimize... query = _response_query.filter(StatementTable.in_response_to.text('%' + _filter + '%')) + query = _response_query.filter(StatementTable.in_response_to is not None) + # else: + # query = _response_query.filter(StatementTable.text_search == _filter) else: _response_query = session.query(ResponseTable) - query = _response_query.filter(ResponseTable.text_search.like('%' + _like + '%')) + query = _response_query.filter(ResponseTable.text_search.like('%' + _filter + '%')) statements.extend(query.all()) @@ -225,6 +248,63 @@ def filter(self, **kwargs): return results + # def filter(self, **kwargs): + # """ + # Returns a list of objects from the database. + # The kwargs parameter can contain any number + # of attributes. Only objects which contain + # all listed attributes and in which all values + # match for all listed attributes will be returned. + # """ + # + # filter_parameters = kwargs.copy() + # + # session = self.__get_session() + # statements = [] + # _response_query = None + # + # if len(filter_parameters) == 0: + # _response_query = session.query(StatementTable) + # statements.extend(_response_query.all()) + # else: + # for fp in filter_parameters: + # _filter = filter_parameters[fp] + # if fp == 'in_response_to' or fp == 'in_response_to__contains': + # if _response_query: + # _response_query.join(StatementTable) + # else: + # _response_query = session.query(StatementTable) + # if isinstance(_filter, list): + # if len(_filter) == 0: + # query = _response_query.filter(StatementTable.in_response_to is None) + # else: + # # _in_response_tables = [] + # # for respnse_table in _like: + # # _in_response_tables.append(get_response_table(respnse_table)) + # query = _response_query.filter(StatementTable.in_response_to.contains(_filter)) + # else: + # query = _response_query.filter(StatementTable.in_response_to.like('%' + _filter + '%')) + # else: + # if fp == 'text' or fp == 'text__contains': # Text always use like + # _response_query = session.query(ResponseTable) + # # if fp == 'text__contains': + # query = _response_query.filter(ResponseTable.text.like('%' + _filter + '%')) + # # if fp == 'text': + # # query = _response_query.filter(ResponseTable.text == _filter) + # + # statements.extend(query.all()) + # + # results = [] + # for statement in statements: + # if isinstance(statement, ResponseTable): + # if statement and statement.statement_table: + # results.append(statement.statement_table.get_statement()) + # else: + # if statement: + # results.append(statement.get_statement()) + # + # return results + def update(self, statement): """ Modifies an entry in the database. @@ -248,10 +328,14 @@ def update(self, statement): else: session.add(get_statement_table(statement)) - if not self.read_only: - session.commit() - else: - session.rollback() + try: + if not self.read_only: + session.commit() + else: + session.rollback() + except DatabaseError as e: + pass + # self.logger.error(statement, str(e.orig)) def get_random(self): """ diff --git a/requirements.txt b/requirements.txt index ee1a656a9..5588f71bc 100644 --- a/requirements.txt +++ b/requirements.txt @@ -4,4 +4,4 @@ nltk>=3.2.0,<4.0.0 pymongo>=3.3.0,<4.0.0 python-twitter>=3.0.0,<4.0.0 textblob>=0.11.0,<0.12.0 -SQLAlchemy>=1.0.14 +SQLAlchemy==1.1.7 \ No newline at end of file diff --git a/setup.cfg b/setup.cfg index 93169dfeb..3c90c4459 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ universal = 1 description-file = readme.md [nosetests] -exclude-dir=tests_django +#exclude-dir=tests_django [flake8] # F812: list comprehension redefines ... diff --git a/tests/base_case.py b/tests/base_case.py index d2f0eed2a..67ef70338 100644 --- a/tests/base_case.py +++ b/tests/base_case.py @@ -40,6 +40,18 @@ def tearDown(self): self.chatbot.storage.drop() + + +class ChatBotSqlAlchemyTestCase(ChatBotTestCase): + + def get_kwargs(self): + kwargs = super(ChatBotSqlAlchemyTestCase, self).get_kwargs() + kwargs['database'] = self.random_string() + kwargs['storage_adapter'] = 'chatterbot.storage.SQLAlchemyDatabaseAdapter' + return kwargs + + + class ChatBotMongoTestCase(ChatBotTestCase): def setUp(self): diff --git a/tests/storage_adapter_tests/integration_tests/sqlalchemy_integration_tests.py b/tests/storage_adapter_tests/integration_tests/sqlalchemy_integration_tests.py new file mode 100644 index 000000000..aa8a50078 --- /dev/null +++ b/tests/storage_adapter_tests/integration_tests/sqlalchemy_integration_tests.py @@ -0,0 +1,17 @@ +from tests.base_case import ChatBotTestCase + + +class SqlAlchemyStorageIntegrationTests(ChatBotTestCase): + + def test_database_is_updated(self): + """ + Test that the database is updated when read_only is set to false. + """ + input_text = 'What is the airspeed velocity of an unladen swallow?' + exists_before = self.chatbot.storage.find(input_text) + + response = self.chatbot.get_response(input_text) + exists_after = self.chatbot.storage.find(input_text) + + self.assertFalse(exists_before) + self.assertTrue(exists_after) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index e6001eafb..84fbd6fe7 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -6,11 +6,10 @@ class SQLAlchemyAdapterTestCase(TestCase): def setUp(self): - self.adapter = SQLAlchemyDatabaseAdapter() + self.adapter = SQLAlchemyDatabaseAdapter(drop_create=True) class SQLAlchemyDatabaseAdapterTestCase(SQLAlchemyAdapterTestCase): - def test_count_returns_zero(self): """ The count method should return a value of 0 diff --git a/tests/teste_main.py b/tests/teste_main.py new file mode 100644 index 000000000..f583bf2f8 --- /dev/null +++ b/tests/teste_main.py @@ -0,0 +1,40 @@ +from chatterbot import ChatBot + +chatbot = ChatBot("Terminal", + + logic_adapters=[ + "chatterbot.logic.BestMatch" + ], + trainer='chatterbot.trainers.ChatterBotCorpusTrainer', + input_adapter="chatterbot.input.TerminalAdapter", + output_adapter="chatterbot.output.TerminalAdapter", + + storage_adapter="chatterbot.storage.SQLAlchemyDatabaseAdapter", + database_uri="sqlite:///database_test.db", # use database_uri or database, database_uri can be especified to choose database driver + database="database_test", # use for sqlite database. Ignored if database_uri especified . + read_only=False, # Readonly database, Default: False + drop_create=True # for recreate database every start read_olny must be False, Default: False + ) + +# Train based on the english corpus +#chatbot.train("chatterbot.corpus.english") + +# Train based on english greetings corpus +chatbot.train("chatterbot.corpus.english.greetings") + +# Train based on the english conversations corpus +# chatbot.train("chatterbot.corpus.english.conversations") + + +print("Type something to begin...") + +# The following loop will execute each time the user enters input +while True: + try: + # We pass None to this method because the parameter + # is not used by the TerminalAdapter + bot_input = chatbot.get_response(None) + + # Press ctrl-c or ctrl-d on the keyboard to exit + except (KeyboardInterrupt, EOFError, SystemExit): + break \ No newline at end of file diff --git a/tests/training_tests/test_chatterbot_corpus_training.py b/tests/training_tests/test_chatterbot_corpus_training.py index 5e2d08f4b..81db5597d 100644 --- a/tests/training_tests/test_chatterbot_corpus_training.py +++ b/tests/training_tests/test_chatterbot_corpus_training.py @@ -30,7 +30,6 @@ def test_train_with_english_corpus(self): self.assertIsNotNone(statement) - class ChatterBotCorpusFilePathTestCase(ChatBotTestCase): def setUp(self): From 80ff870d62a3571e1a45124d6077c7ff1f8960c2 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 00:32:06 -0300 Subject: [PATCH 27/43] Clean up code. --- chatterbot/storage/sqlalchemy.py | 1 - tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 2 +- 2 files changed, 1 insertion(+), 2 deletions(-) diff --git a/chatterbot/storage/sqlalchemy.py b/chatterbot/storage/sqlalchemy.py index 99ad7be41..f23089e43 100644 --- a/chatterbot/storage/sqlalchemy.py +++ b/chatterbot/storage/sqlalchemy.py @@ -18,7 +18,6 @@ Base = declarative_base() - class StatementTable(Base): __tablename__ = 'StatementTable' diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index 84fbd6fe7..9c6f0b140 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -230,7 +230,7 @@ def test_filter_equal_results(self): self.assertIn(statement1, results) self.assertIn(statement2, results) - def test_filter_contains_result(self): + def test_filter_contains_result(self): self.adapter.update(self.statement1) self.adapter.update(self.statement2) From 9fa94a38be9d3a629f32ecc922179992f71c97d1 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 00:33:43 -0300 Subject: [PATCH 28/43] ignore venv* --- .gitignore | 1 + 1 file changed, 1 insertion(+) diff --git a/.gitignore b/.gitignore index 412e41634..69391cda8 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,4 @@ examples/settings.py examples/ubuntu_dialogs* .env .out +venv* From 6b8a7d260964d3b6dd3c4eb503f05db4a58ca02b Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 01:19:52 -0300 Subject: [PATCH 29/43] Last fix check before PR --- chatterbot/storage/sqlalchemy.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/chatterbot/storage/sqlalchemy.py b/chatterbot/storage/sqlalchemy.py index f23089e43..3d1367ef0 100644 --- a/chatterbot/storage/sqlalchemy.py +++ b/chatterbot/storage/sqlalchemy.py @@ -18,6 +18,7 @@ Base = declarative_base() + class StatementTable(Base): __tablename__ = 'StatementTable' @@ -211,7 +212,7 @@ def filter(self, **kwargs): _response_query = session.query(StatementTable) statements.extend(_response_query.all()) else: - for fp in filter_parameters: + for i, fp in enumerate(filter_parameters): _filter = filter_parameters[fp] if fp in ['in_response_to', 'in_response_to__contains']: _response_query = session.query(StatementTable) @@ -224,17 +225,16 @@ def filter(self, **kwargs): query = _response_query.filter( StatementTable.in_response_to.contains(get_response_table(f))) else: - # if fp == 'in_response_to__contains': - # FIXME Optimize... query = _response_query.filter(StatementTable.in_response_to.text('%' + _filter + '%')) - query = _response_query.filter(StatementTable.in_response_to is not None) - # else: - # query = _response_query.filter(StatementTable.text_search == _filter) - + if fp == 'in_response_to__contains': + query = _response_query.join(ResponseTable).filter(ResponseTable.text == _filter) + else: + query = _response_query.filter(StatementTable.in_response_to == None) else: _response_query = session.query(ResponseTable) query = _response_query.filter(ResponseTable.text_search.like('%' + _filter + '%')) - statements.extend(query.all()) + if len(filter_parameters) == i + 1: + statements.extend(query.all()) results = [] for statement in statements: From b39ca44f8d363b95ddfb5206ca12569fec13515a Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 08:32:47 -0300 Subject: [PATCH 30/43] Fix test-requirements --- test-requirements.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/test-requirements.txt b/test-requirements.txt index 9dce2d73c..e32e6abbf 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -8,3 +8,5 @@ nose-exclude>=0.5.0,<0.6.0 twython sphinx sphinx_rtd_theme +textblob>=0.11.0,<0.12.0 +SQLAlchemy==1.1.7 \ No newline at end of file From 18c3d69ae1560a7e0da7a85227b45e619dc30b7e Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 08:43:36 -0300 Subject: [PATCH 31/43] Renamed file, import conflicts in CI --- chatterbot/storage/{sqlalchemy.py => sqlalchemy_storage.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename chatterbot/storage/{sqlalchemy.py => sqlalchemy_storage.py} (100%) diff --git a/chatterbot/storage/sqlalchemy.py b/chatterbot/storage/sqlalchemy_storage.py similarity index 100% rename from chatterbot/storage/sqlalchemy.py rename to chatterbot/storage/sqlalchemy_storage.py From e015ea6ae5cbbfe59ad933da93dc58042e390f48 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 09:00:33 -0300 Subject: [PATCH 32/43] Fix import rename. --- chatterbot/storage/__init__.py | 2 +- tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/chatterbot/storage/__init__.py b/chatterbot/storage/__init__.py index 658d933a7..2efe59951 100644 --- a/chatterbot/storage/__init__.py +++ b/chatterbot/storage/__init__.py @@ -2,4 +2,4 @@ from .django_storage import DjangoStorageAdapter from .jsonfile import JsonFileStorageAdapter from .mongodb import MongoDatabaseAdapter -from .sqlalchemy import SQLAlchemyDatabaseAdapter +from .sqlalchemy_storage import SQLAlchemyDatabaseAdapter diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index 9c6f0b140..ab31ed988 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -1,7 +1,7 @@ from unittest import TestCase from chatterbot.conversation import Statement, Response -from chatterbot.storage.sqlalchemy import SQLAlchemyDatabaseAdapter +from chatterbot.storage.sqlalchemy_storage import SQLAlchemyDatabaseAdapter class SQLAlchemyAdapterTestCase(TestCase): @@ -230,7 +230,7 @@ def test_filter_equal_results(self): self.assertIn(statement1, results) self.assertIn(statement2, results) - def test_filter_contains_result(self): + def test_filter_contains_result(self): self.adapter.update(self.statement1) self.adapter.update(self.statement2) From 44ed12d7bee40824c6216ed8247699adf766a4f6 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 09:09:00 -0300 Subject: [PATCH 33/43] Travis first install ChatterBot after install requirements... --- chatterbot/storage/__init__.py | 1 - 1 file changed, 1 deletion(-) diff --git a/chatterbot/storage/__init__.py b/chatterbot/storage/__init__.py index 2efe59951..2fe7e08b9 100644 --- a/chatterbot/storage/__init__.py +++ b/chatterbot/storage/__init__.py @@ -2,4 +2,3 @@ from .django_storage import DjangoStorageAdapter from .jsonfile import JsonFileStorageAdapter from .mongodb import MongoDatabaseAdapter -from .sqlalchemy_storage import SQLAlchemyDatabaseAdapter From 70caa0becd2b5a890e4ec2bfaaf6a84125abd010 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 09:19:10 -0300 Subject: [PATCH 34/43] Fiz import --- chatterbot/storage/__init__.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/chatterbot/storage/__init__.py b/chatterbot/storage/__init__.py index 2fe7e08b9..644421cfc 100644 --- a/chatterbot/storage/__init__.py +++ b/chatterbot/storage/__init__.py @@ -2,3 +2,9 @@ from .django_storage import DjangoStorageAdapter from .jsonfile import JsonFileStorageAdapter from .mongodb import MongoDatabaseAdapter + +# FIXME Better way manage import +try: + from .sqlalchemy_storage import SQLAlchemyDatabaseAdapter +except ImportError: + pass From 5c8d7b6df62b4a136a8f109a584c2320faa1776f Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 09:22:54 -0300 Subject: [PATCH 35/43] Fiz imports --- test-requirements.txt | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/test-requirements.txt b/test-requirements.txt index e32e6abbf..8cb31b5c1 100644 --- a/test-requirements.txt +++ b/test-requirements.txt @@ -7,6 +7,4 @@ nose nose-exclude>=0.5.0,<0.6.0 twython sphinx -sphinx_rtd_theme -textblob>=0.11.0,<0.12.0 -SQLAlchemy==1.1.7 \ No newline at end of file +sphinx_rtd_theme \ No newline at end of file From d0b68b897581f575135f05ae4bb1980b6769071e Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 10:12:35 -0300 Subject: [PATCH 36/43] Removed temp test class. --- tests/teste_main.py | 40 ---------------------------------------- 1 file changed, 40 deletions(-) delete mode 100644 tests/teste_main.py diff --git a/tests/teste_main.py b/tests/teste_main.py deleted file mode 100644 index f583bf2f8..000000000 --- a/tests/teste_main.py +++ /dev/null @@ -1,40 +0,0 @@ -from chatterbot import ChatBot - -chatbot = ChatBot("Terminal", - - logic_adapters=[ - "chatterbot.logic.BestMatch" - ], - trainer='chatterbot.trainers.ChatterBotCorpusTrainer', - input_adapter="chatterbot.input.TerminalAdapter", - output_adapter="chatterbot.output.TerminalAdapter", - - storage_adapter="chatterbot.storage.SQLAlchemyDatabaseAdapter", - database_uri="sqlite:///database_test.db", # use database_uri or database, database_uri can be especified to choose database driver - database="database_test", # use for sqlite database. Ignored if database_uri especified . - read_only=False, # Readonly database, Default: False - drop_create=True # for recreate database every start read_olny must be False, Default: False - ) - -# Train based on the english corpus -#chatbot.train("chatterbot.corpus.english") - -# Train based on english greetings corpus -chatbot.train("chatterbot.corpus.english.greetings") - -# Train based on the english conversations corpus -# chatbot.train("chatterbot.corpus.english.conversations") - - -print("Type something to begin...") - -# The following loop will execute each time the user enters input -while True: - try: - # We pass None to this method because the parameter - # is not used by the TerminalAdapter - bot_input = chatbot.get_response(None) - - # Press ctrl-c or ctrl-d on the keyboard to exit - except (KeyboardInterrupt, EOFError, SystemExit): - break \ No newline at end of file From ba6dbb240eb2fd75b57071a44859e5f66c20c5d5 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 19:46:35 -0300 Subject: [PATCH 37/43] fix import. --- chatterbot/storage/sqlalchemy_storage.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/chatterbot/storage/sqlalchemy_storage.py b/chatterbot/storage/sqlalchemy_storage.py index 3d1367ef0..396622a5a 100644 --- a/chatterbot/storage/sqlalchemy_storage.py +++ b/chatterbot/storage/sqlalchemy_storage.py @@ -1,7 +1,7 @@ import json import random -import sqlalchemy + from sqlalchemy import Column, ForeignKey from sqlalchemy import PickleType from sqlalchemy import String From be8d2bb8f66559f494913233540b5ffba86de49b Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Wed, 19 Apr 2017 20:03:36 -0300 Subject: [PATCH 38/43] Rolback changes. --- setup.cfg | 2 +- tests/base_case.py | 14 +------------- .../test_chatterbot_corpus_training.py | 3 ++- 3 files changed, 4 insertions(+), 15 deletions(-) diff --git a/setup.cfg b/setup.cfg index 3c90c4459..93169dfeb 100644 --- a/setup.cfg +++ b/setup.cfg @@ -8,7 +8,7 @@ universal = 1 description-file = readme.md [nosetests] -#exclude-dir=tests_django +exclude-dir=tests_django [flake8] # F812: list comprehension redefines ... diff --git a/tests/base_case.py b/tests/base_case.py index 67ef70338..4a1947c9e 100644 --- a/tests/base_case.py +++ b/tests/base_case.py @@ -40,18 +40,6 @@ def tearDown(self): self.chatbot.storage.drop() - - -class ChatBotSqlAlchemyTestCase(ChatBotTestCase): - - def get_kwargs(self): - kwargs = super(ChatBotSqlAlchemyTestCase, self).get_kwargs() - kwargs['database'] = self.random_string() - kwargs['storage_adapter'] = 'chatterbot.storage.SQLAlchemyDatabaseAdapter' - return kwargs - - - class ChatBotMongoTestCase(ChatBotTestCase): def setUp(self): @@ -74,4 +62,4 @@ def get_kwargs(self): kwargs = super(ChatBotMongoTestCase, self).get_kwargs() kwargs['database'] = self.random_string() kwargs['storage_adapter'] = 'chatterbot.storage.MongoDatabaseAdapter' - return kwargs + return kwargs \ No newline at end of file diff --git a/tests/training_tests/test_chatterbot_corpus_training.py b/tests/training_tests/test_chatterbot_corpus_training.py index 81db5597d..7f19c5232 100644 --- a/tests/training_tests/test_chatterbot_corpus_training.py +++ b/tests/training_tests/test_chatterbot_corpus_training.py @@ -30,6 +30,7 @@ def test_train_with_english_corpus(self): self.assertIsNotNone(statement) + class ChatterBotCorpusFilePathTestCase(ChatBotTestCase): def setUp(self): @@ -68,4 +69,4 @@ def test_train_with_english_corpus_training_slash(self): self.chatbot.train(file_path) statement = self.chatbot.storage.find('Hello') - self.assertIsNotNone(statement) + self.assertIsNotNone(statement) \ No newline at end of file From 401ba7a5cb9e15cc04d10a2d4f374a551383370e Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Thu, 20 Apr 2017 12:02:07 -0300 Subject: [PATCH 39/43] Random database name for tests --- .../test_sqlalchemy_adapter.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index ab31ed988..3f5ee2b03 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -6,7 +6,18 @@ class SQLAlchemyAdapterTestCase(TestCase): def setUp(self): - self.adapter = SQLAlchemyDatabaseAdapter(drop_create=True) + """ + Instantiate the adapter. + """ + from random import randint + + # Generate a random name for the database + database_name = str(randint(0, 9000)) + + self.adapter = SQLAlchemyDatabaseAdapter( + database='sqlite_' + database_name, + drop_create=True + ) class SQLAlchemyDatabaseAdapterTestCase(SQLAlchemyAdapterTestCase): From 2e880873b098adafbb90239a9b09d238372a5dd5 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Thu, 20 Apr 2017 13:45:04 -0300 Subject: [PATCH 40/43] Fix filter method, order of filters returning wrong values. --- chatterbot/storage/sqlalchemy_storage.py | 24 ++++++++++++------- .../test_sqlalchemy_adapter.py | 4 ++-- 2 files changed, 17 insertions(+), 11 deletions(-) diff --git a/chatterbot/storage/sqlalchemy_storage.py b/chatterbot/storage/sqlalchemy_storage.py index 396622a5a..608dff71f 100644 --- a/chatterbot/storage/sqlalchemy_storage.py +++ b/chatterbot/storage/sqlalchemy_storage.py @@ -1,7 +1,6 @@ import json import random - from sqlalchemy import Column, ForeignKey from sqlalchemy import PickleType from sqlalchemy import String @@ -207,7 +206,8 @@ def filter(self, **kwargs): session = self.__get_session() statements = [] - + # _response_query = None + _query = None if len(filter_parameters) == 0: _response_query = session.query(StatementTable) statements.extend(_response_query.all()) @@ -218,25 +218,31 @@ def filter(self, **kwargs): _response_query = session.query(StatementTable) if isinstance(_filter, list): if len(_filter) == 0: - query = _response_query.filter( + _query = _response_query.filter( StatementTable.in_response_to == None) # Here must use == instead of is else: for f in _filter: - query = _response_query.filter( + _query = _response_query.filter( StatementTable.in_response_to.contains(get_response_table(f))) else: if fp == 'in_response_to__contains': - query = _response_query.join(ResponseTable).filter(ResponseTable.text == _filter) + _query = _response_query.join(ResponseTable).filter(ResponseTable.text == _filter) else: - query = _response_query.filter(StatementTable.in_response_to == None) + _query = _response_query.filter(StatementTable.in_response_to == None) else: - _response_query = session.query(ResponseTable) - query = _response_query.filter(ResponseTable.text_search.like('%' + _filter + '%')) + if _query: + _query = _query.filter(ResponseTable.text_search.like('%' + _filter + '%')) + else: + _response_query = session.query(ResponseTable) + _query = _response_query.filter(ResponseTable.text_search.like('%' + _filter + '%')) + if _query is None: + return [] if len(filter_parameters) == i + 1: - statements.extend(query.all()) + statements.extend(_query.all()) results = [] + for statement in statements: if isinstance(statement, ResponseTable): if statement and statement.statement_table: diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index 3f5ee2b03..c0368e2b7 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -264,8 +264,8 @@ def test_filter_multiple_parameters(self): self.adapter.update(self.statement2) results = self.adapter.filter( - text="Testing...", - in_response_to__contains="Why are you counting?" + in_response_to__contains="Why are you counting?", + text="Testing..." ) self.assertEqual(len(results), 1) From 3af0b4115f0cea044f16deadc55ff626790e11ed Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Thu, 20 Apr 2017 13:46:27 -0300 Subject: [PATCH 41/43] Fix filter method, order of filters returning wrong values. --- tests/storage_adapter_tests/test_sqlalchemy_adapter.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py index c0368e2b7..3cd781726 100644 --- a/tests/storage_adapter_tests/test_sqlalchemy_adapter.py +++ b/tests/storage_adapter_tests/test_sqlalchemy_adapter.py @@ -264,8 +264,8 @@ def test_filter_multiple_parameters(self): self.adapter.update(self.statement2) results = self.adapter.filter( - in_response_to__contains="Why are you counting?", - text="Testing..." + text="Testing...", + in_response_to__contains = "Why are you counting?" ) self.assertEqual(len(results), 1) From 1c01393b6c7bef184fd7a3fefd9fecbc0146b820 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Mon, 24 Apr 2017 11:04:50 -0300 Subject: [PATCH 42/43] Changes by code review: Clean up, removed unnecessary code comments and imports in requirements.txt, change imports in sqlalchemy and a little refactor. --- chatterbot/storage/__init__.py | 7 +- chatterbot/storage/sqlalchemy_storage.py | 154 +++++------------------ requirements.txt | 1 - 3 files changed, 32 insertions(+), 130 deletions(-) diff --git a/chatterbot/storage/__init__.py b/chatterbot/storage/__init__.py index 644421cfc..2efe59951 100644 --- a/chatterbot/storage/__init__.py +++ b/chatterbot/storage/__init__.py @@ -2,9 +2,4 @@ from .django_storage import DjangoStorageAdapter from .jsonfile import JsonFileStorageAdapter from .mongodb import MongoDatabaseAdapter - -# FIXME Better way manage import -try: - from .sqlalchemy_storage import SQLAlchemyDatabaseAdapter -except ImportError: - pass +from .sqlalchemy_storage import SQLAlchemyDatabaseAdapter diff --git a/chatterbot/storage/sqlalchemy_storage.py b/chatterbot/storage/sqlalchemy_storage.py index 608dff71f..c9469b28d 100644 --- a/chatterbot/storage/sqlalchemy_storage.py +++ b/chatterbot/storage/sqlalchemy_storage.py @@ -1,24 +1,23 @@ import json import random -from sqlalchemy import Column, ForeignKey -from sqlalchemy import PickleType -from sqlalchemy import String -from sqlalchemy import Integer -from sqlalchemy import create_engine -from sqlalchemy.exc import DatabaseError -from sqlalchemy.ext.declarative import declarative_base -from sqlalchemy.orm import relationship -from sqlalchemy.orm import sessionmaker - from chatterbot.storage import StorageAdapter from chatterbot.conversation import Response from chatterbot.conversation import Statement -Base = declarative_base() +_base = None +try: + from sqlalchemy.ext.declarative import declarative_base + + _base = declarative_base() +except: + pass -class StatementTable(Base): +class StatementTable(_base): + from sqlalchemy import Column, Integer, String, PickleType + from sqlalchemy.orm import relationship + __tablename__ = 'StatementTable' def get_statement(self): @@ -40,7 +39,9 @@ def get_statement_serialized(context): text_search = Column(String, primary_key=True, default=get_statement_serialized) -class ResponseTable(Base): +class ResponseTable(_base): + from sqlalchemy import Column, Integer, String, ForeignKey + from sqlalchemy.orm import relationship __tablename__ = 'ResponseTable' def get_reponse_serialized(context): @@ -53,8 +54,6 @@ def get_reponse_serialized(context): occurrence = Column(Integer) statement_text = Column(String, ForeignKey('StatementTable.text')) - # Old: statement_table = relationship("StatementTable", backref=backref('in_response_to'), cascade="all, delete-orphan", single_parent=True) - # Test relationship: statement_table = relationship("StatementTable", back_populates="in_response_to", cascade="all", uselist=False) text_search = Column(String, primary_key=True, default=get_reponse_serialized) @@ -63,14 +62,6 @@ def get_response(self): return Response(text=self.text, **occ) -# # relational table TO REMOVE -# in_response_to = sqlalchemy.Table('in_responses_to', -# Base.metadata, -# sqlalchemy.Column('stmt_id', sqlalchemy.Integer, ForeignKey('StatementTable.id')), -# sqlalchemy.Column('resp_id', sqlalchemy.Integer, -# sqlalchemy.ForeignKey('ResponseTable.id'))) - - def get_statement_table(statement): responses = list(map(get_response_table, statement.in_response_to)) return StatementTable(text=statement.text, in_response_to=responses, extra_data=statement.extra_data) @@ -87,11 +78,13 @@ class SQLAlchemyDatabaseAdapter(StorageAdapter): def __init__(self, **kwargs): super(SQLAlchemyDatabaseAdapter, self).__init__(**kwargs) + from sqlalchemy import create_engine + self.database_name = self.kwargs.get( "database", "chatterbot-database" ) - # if some annoing blank space wrong... + # if some annoying blank space wrong... db_name = self.database_name.strip() # default uses sqlite @@ -101,32 +94,6 @@ def __init__(self, **kwargs): self.engine = create_engine(self.database_uri) - # metadata = MetaData(self.engine) - - # Recreate database - # metadata.drop_all() - # - # self.response = Table('response', metadata, - # Column('id', Integer, primary_key=True), - # Column('text', Text), - # Column('occurrence', Text), - # ) - # - # self.statement = Table('statement', metadata, - # Column('id', Integer, primary_key=True), - # Column('text', Text), - # Column('in_response_to', Integer, ForeignKey('response.id')), - # Column('extra_data', Text) - # ) - # - # # mapper(Response, self.response, - # # # non_primary=True, - # # properties={ - # # 'statement': relationship(Statement, backref='response') - # # }, ) - # mapper(Statement, self.statement) - # mapper(Response, self.response) - self.read_only = self.kwargs.get( "read_only", False ) @@ -150,6 +117,8 @@ def __get_session(self): """ :rtype: Session """ + from sqlalchemy.orm import sessionmaker + Session = sessionmaker(bind=self.engine) session = Session() return session @@ -185,13 +154,7 @@ def remove(self, statement_text): record = query.first() session.delete(record) - try: - if not self.read_only: - session.commit() - else: - session.rollback() - except DatabaseError as e: - self.logger.error(statement_text, str(e.orig)) + self._session_finish(session, statement_text) def filter(self, **kwargs): """ @@ -253,69 +216,11 @@ def filter(self, **kwargs): return results - # def filter(self, **kwargs): - # """ - # Returns a list of objects from the database. - # The kwargs parameter can contain any number - # of attributes. Only objects which contain - # all listed attributes and in which all values - # match for all listed attributes will be returned. - # """ - # - # filter_parameters = kwargs.copy() - # - # session = self.__get_session() - # statements = [] - # _response_query = None - # - # if len(filter_parameters) == 0: - # _response_query = session.query(StatementTable) - # statements.extend(_response_query.all()) - # else: - # for fp in filter_parameters: - # _filter = filter_parameters[fp] - # if fp == 'in_response_to' or fp == 'in_response_to__contains': - # if _response_query: - # _response_query.join(StatementTable) - # else: - # _response_query = session.query(StatementTable) - # if isinstance(_filter, list): - # if len(_filter) == 0: - # query = _response_query.filter(StatementTable.in_response_to is None) - # else: - # # _in_response_tables = [] - # # for respnse_table in _like: - # # _in_response_tables.append(get_response_table(respnse_table)) - # query = _response_query.filter(StatementTable.in_response_to.contains(_filter)) - # else: - # query = _response_query.filter(StatementTable.in_response_to.like('%' + _filter + '%')) - # else: - # if fp == 'text' or fp == 'text__contains': # Text always use like - # _response_query = session.query(ResponseTable) - # # if fp == 'text__contains': - # query = _response_query.filter(ResponseTable.text.like('%' + _filter + '%')) - # # if fp == 'text': - # # query = _response_query.filter(ResponseTable.text == _filter) - # - # statements.extend(query.all()) - # - # results = [] - # for statement in statements: - # if isinstance(statement, ResponseTable): - # if statement and statement.statement_table: - # results.append(statement.statement_table.get_statement()) - # else: - # if statement: - # results.append(statement.get_statement()) - # - # return results - def update(self, statement): """ Modifies an entry in the database. Creates an entry if one does not exist. """ - session = self.__get_session() if statement: query = self.__statement_filter(session, **{"text": statement.text}) @@ -333,14 +238,7 @@ def update(self, statement): else: session.add(get_statement_table(statement)) - try: - if not self.read_only: - session.commit() - else: - session.rollback() - except DatabaseError as e: - pass - # self.logger.error(statement, str(e.orig)) + self._session_finish(session) def get_random(self): """ @@ -361,3 +259,13 @@ def drop(self): Drop the database attached to a given adapter. """ Base.metadata.drop_all(self.engine) + + def _session_finish(self, session, statement_text=None): + from sqlalchemy.exc import DatabaseError + try: + if not self.read_only: + session.commit() + else: + session.rollback() + except DatabaseError as e: + self.logger.error(statement_text, str(e.orig)) diff --git a/requirements.txt b/requirements.txt index 5588f71bc..8f28c03b3 100644 --- a/requirements.txt +++ b/requirements.txt @@ -3,5 +3,4 @@ jsondatabase>=0.1.7,<1.0.0 nltk>=3.2.0,<4.0.0 pymongo>=3.3.0,<4.0.0 python-twitter>=3.0.0,<4.0.0 -textblob>=0.11.0,<0.12.0 SQLAlchemy==1.1.7 \ No newline at end of file From 6ffb8476a94b339c4f82943cf3927874d1fc5861 Mon Sep 17 00:00:00 2001 From: "Davi S. Zucon" Date: Mon, 24 Apr 2017 11:26:30 -0300 Subject: [PATCH 43/43] Fixing import, not depend on SQLAlchemy. --- chatterbot/storage/sqlalchemy_storage.py | 84 ++++++++++++------------ 1 file changed, 43 insertions(+), 41 deletions(-) diff --git a/chatterbot/storage/sqlalchemy_storage.py b/chatterbot/storage/sqlalchemy_storage.py index c9469b28d..1c2fcbeb7 100644 --- a/chatterbot/storage/sqlalchemy_storage.py +++ b/chatterbot/storage/sqlalchemy_storage.py @@ -6,60 +6,62 @@ from chatterbot.conversation import Statement _base = None + try: from sqlalchemy.ext.declarative import declarative_base _base = declarative_base() -except: - pass -class StatementTable(_base): - from sqlalchemy import Column, Integer, String, PickleType - from sqlalchemy.orm import relationship + class StatementTable(_base): + from sqlalchemy import Column, Integer, String, PickleType + from sqlalchemy.orm import relationship - __tablename__ = 'StatementTable' + __tablename__ = 'StatementTable' - def get_statement(self): - stmt = Statement(self.text, **self.extra_data) - for resp in self.in_response_to: - stmt.add_response(resp.get_response()) - return stmt + def get_statement(self): + stmt = Statement(self.text, **self.extra_data) + for resp in self.in_response_to: + stmt.add_response(resp.get_response()) + return stmt - def get_statement_serialized(context): - params = context.current_parameters - del (params['text_search']) - return json.dumps(params) + def get_statement_serialized(context): + params = context.current_parameters + del (params['text_search']) + return json.dumps(params) - id = Column(Integer) - text = Column(String, primary_key=True) - extra_data = Column(PickleType) - # relationship: - in_response_to = relationship("ResponseTable", back_populates="statement_table") - text_search = Column(String, primary_key=True, default=get_statement_serialized) + id = Column(Integer) + text = Column(String, primary_key=True) + extra_data = Column(PickleType) + # relationship: + in_response_to = relationship("ResponseTable", back_populates="statement_table") + text_search = Column(String, primary_key=True, default=get_statement_serialized) -class ResponseTable(_base): - from sqlalchemy import Column, Integer, String, ForeignKey - from sqlalchemy.orm import relationship - __tablename__ = 'ResponseTable' + class ResponseTable(_base): + from sqlalchemy import Column, Integer, String, ForeignKey + from sqlalchemy.orm import relationship + __tablename__ = 'ResponseTable' - def get_reponse_serialized(context): - params = context.current_parameters - del (params['text_search']) - return json.dumps(params) + def get_reponse_serialized(context): + params = context.current_parameters + del (params['text_search']) + return json.dumps(params) - id = Column(Integer) - text = Column(String, primary_key=True) - occurrence = Column(Integer) - statement_text = Column(String, ForeignKey('StatementTable.text')) + id = Column(Integer) + text = Column(String, primary_key=True) + occurrence = Column(Integer) + statement_text = Column(String, ForeignKey('StatementTable.text')) - statement_table = relationship("StatementTable", back_populates="in_response_to", cascade="all", uselist=False) - text_search = Column(String, primary_key=True, default=get_reponse_serialized) + statement_table = relationship("StatementTable", back_populates="in_response_to", cascade="all", uselist=False) + text_search = Column(String, primary_key=True, default=get_reponse_serialized) - def get_response(self): - occ = {"occurrence": self.occurrence} - return Response(text=self.text, **occ) + def get_response(self): + occ = {"occurrence": self.occurrence} + return Response(text=self.text, **occ) + +except ImportError: + pass def get_statement_table(statement): @@ -103,8 +105,8 @@ def __init__(self, **kwargs): ) if not self.read_only and self.drop_create: - Base.metadata.drop_all(self.engine) - Base.metadata.create_all(self.engine) + _base.metadata.drop_all(self.engine) + _base.metadata.create_all(self.engine) def count(self): """ @@ -258,7 +260,7 @@ def drop(self): """ Drop the database attached to a given adapter. """ - Base.metadata.drop_all(self.engine) + _base.metadata.drop_all(self.engine) def _session_finish(self, session, statement_text=None): from sqlalchemy.exc import DatabaseError