From 4aa746aab32e58c6e62c1be81262dd3db6ed0d37 Mon Sep 17 00:00:00 2001 From: Wolfgang Fahl Date: Fri, 16 Jul 2021 14:30:22 +0200 Subject: [PATCH] improves #243 --- migration/openresearch/event.py | 75 +++++++++++++------ migration/requirements.txt | 2 +- migration/tests/testIssue155__OOAccess.py | 3 +- migration/tests/testIssue243_Refactoring.py | 25 ++----- migration/tests/test_issue119_OrdinalFixer.py | 3 + 5 files changed, 65 insertions(+), 43 deletions(-) diff --git a/migration/openresearch/event.py b/migration/openresearch/event.py index fc75bc8..7b70b63 100644 --- a/migration/openresearch/event.py +++ b/migration/openresearch/event.py @@ -11,6 +11,8 @@ from wikibot.wikiuser import WikiUser from wikibot.wikiclient import WikiClient from wikibot.wikipush import WikiPush +from wikifile.wikiFileManager import WikiFileManager +from wikifile.wikiFile import WikiFile import os import time @@ -24,6 +26,21 @@ def __init__(self): ''' Constructor ''' + pass + + def __str__(self): + ''' + return my + ''' + text=self.__class__.__name__ + attrs=["pageTitle","acronym","title"] + delim=":" + for attr in attrs: + if hasattr(self, attr): + value=getattr(self,attr) + text+=f"{delim}{value}" + delim=":" + return text @staticmethod def fromWikiSonToLod(record:dict,lookup:dict)->dict: @@ -202,10 +219,11 @@ def fromLoD(self,lod): # call the constructor to get a new instance try: entity=self.clazz() - if hasattr(entity,"fixRecord"): - fixRecord=getattr(entity,'fixRecord'); - if callable(fixRecord): - fixRecord(record) + # TODO callBacks should be more generic (if any) + #if hasattr(entity,"fixRecord"): + # fixRecord=getattr(entity,'fixRecord'); + # if callable(fixRecord): + # fixRecord(record) entity.fromDict(record) entityList.append(entity) except Exception as ex: @@ -223,9 +241,16 @@ def fromWikiFileManager(self,wikiFileManager): initialize me from the given WikiFileManager """ self.wikiFileManager= wikiFileManager - wikiFileList=wikiFileManager.getAllWikiFiles() - lod=self.wikiFileManager.convertWikiFilesToLOD(wikiFileList.values(),self.getEntityName()) - self.normalizeLodFromWikiSonToLod(lod) + wikiFileDict=wikiFileManager.getAllWikiFiles() + self.fromWikiFiles(wikiFileDict.values()) + + def fromWikiFiles(self,wikiFileList:list): + ''' + initialize me from the given list of wiki files + ''' + templateName=self.clazz.templateName + wikiSonLod=WikiFileManager.convertWikiFilesToLOD(wikiFileList,templateName) + lod=self.normalizeLodFromWikiSonToLod(wikiSonLod) self.fromLoD(lod) @classmethod @@ -242,6 +267,19 @@ def getPropertyLookup(cls)->dict: lookup,_duplicates=LOD.getLookup(propertyLookupList, 'prop') return lookup + def fromSampleWikiSonLod(self,entityClass): + ''' + get a list of dicts derived form the wikiSonSamples + + Returns: + list: a list of dicts for my sampleWikiText in WikiSon notation + ''' + wikiFileList=[] + for sampleWikiText in entityClass.getSampleWikiTextList(): + pageTitle=None + wikiFile=WikiFile(name=pageTitle,wikiText=sampleWikiText) + wikiFileList.append(wikiFile) + self.fromWikiFiles(wikiFileList) @classmethod def normalizeLodFromWikiSonToLod(cls, wikiSonRecords:list)->list: @@ -329,7 +367,12 @@ def __init__(self): class EventSeries(OREntity): ''' + an event Series ''' + # TODO - this is the legacy templateName - make sure after / during migration + # this is handled properly + templateName="Event series" + def __init__(self): ''' Constructor @@ -390,13 +433,6 @@ def getSampleWikiTextList(cls, mode='legacy'): samplesWikiSon = "..." return samplesWikiSon - - - def __str__(self): - text=self.pageTitle - if hasattr(self, "acronym"): - text+="(%s)" %self.acronym - return text class EventList(OREntityList): propertyLookupList=[ @@ -432,6 +468,8 @@ class Event(OREntity): see https://rq.bitplan.com/index.php/Event ''' + templateName="Event" + def __init__(self): ''' Constructor @@ -538,15 +576,6 @@ def getSampleWikiTextList(cls,mode='legacy'): return samplesWikiSon - - - - def __str__(self): - text=self.pageTitle - if hasattr(self, "acronym"): - text+="(%s)" %self.acronym - return text - class CountryList(OREntityList): ''' a list of countries diff --git a/migration/requirements.txt b/migration/requirements.txt index 92cc79b..985b153 100644 --- a/migration/requirements.txt +++ b/migration/requirements.txt @@ -7,6 +7,6 @@ py-3rdparty-mediawiki>=0.4.8 # https://github.com/5j9/wikitextparser wikitextparser>=0.47.4 # https://pypi.org/project/wikirender/ -wikirender>=0.0.20 +wikirender>=0.0.22 # https://github.com/somnathrakshit/geograpy3 geograpy3>=0.1.29 \ No newline at end of file diff --git a/migration/tests/testIssue155__OOAccess.py b/migration/tests/testIssue155__OOAccess.py index 3b8e761..a819cc4 100644 --- a/migration/tests/testIssue155__OOAccess.py +++ b/migration/tests/testIssue155__OOAccess.py @@ -5,7 +5,6 @@ ''' import unittest from openresearch.event import Event, EventList, EventSeries, EventSeriesList -from lodstorage.jsonable import Types from lodstorage.sql import SQLDB from ormigrate.toolbox import HelperFunctions as hf, Profiler @@ -27,7 +26,7 @@ def testEventSeries(self): test eventseries handling ''' samples= EventSeries.getSamples() - wikiSonRecords=EventSeries.getSampleWikiSon() + wikiSonRecords=EventSeriesList.getSampleWikiSonLod() wikiSonLod=EventSeriesList.normalizeLodFromWikiSonToLod(wikiSonRecords) for lod in (samples,wikiSonLod): eventSeriesList=EventSeriesList() diff --git a/migration/tests/testIssue243_Refactoring.py b/migration/tests/testIssue243_Refactoring.py index 3bf98e6..234baae 100644 --- a/migration/tests/testIssue243_Refactoring.py +++ b/migration/tests/testIssue243_Refactoring.py @@ -6,7 +6,6 @@ import unittest from openresearch.eventcorpus import EventList,EventSeriesList from openresearch.event import EventSeries,Event -from wikifile.wikiFile import WikiFile class TestRefactoring(unittest.TestCase): ''' @@ -26,24 +25,16 @@ def testWikiSonToLodPropertyMapping(self): ''' test the mapping ''' - for entityList,entity,templateName in [ - (EventList,Event,"Event"), - (EventSeriesList,EventSeries,"Event series") - ]: + for entityListClass,entityClass in (EventList,Event),(EventSeriesList,EventSeries): + entityList=entityListClass() + entityList.fromSampleWikiSonLod(entityClass) #propertyLookup=entityList.getPropertyLookup() #print(propertyLookup) - listOfSampleWikiSon=entity.getSampleWikiSon() - wikiSonRecords=[] - for sampleWikiSon in listOfSampleWikiSon: - wikiFile=WikiFile(name="noname",wikiText=sampleWikiSon) - #print(str(wikiFile)) - record=wikiFile.extract_template(templateName) - if self.debug: - print(record) - wikiSonRecords.append(record) - lod=entityList.normalizeLodFromWikiSonToLod(wikiSonRecords) - if self.debug: - print(lod) + self.assertTrue(len(entityList.getList())>0) + for entity in entityList.getList(): + self.assertTrue(isinstance(entity,entityClass)) + if self.debug: + print(entity) pass diff --git a/migration/tests/test_issue119_OrdinalFixer.py b/migration/tests/test_issue119_OrdinalFixer.py index 16b3bb0..9bc2f56 100644 --- a/migration/tests/test_issue119_OrdinalFixer.py +++ b/migration/tests/test_issue119_OrdinalFixer.py @@ -18,6 +18,7 @@ class TestOrdinalFixer(unittest.TestCase): def setUp(self): self.testAll=True + self.debug=False pass @@ -51,6 +52,8 @@ def testOrdinalFixer(self): debug=self.debug if debug: print (painCounter) + if pageList is None: + self.assertTrue(painCounter[5]>800) def testOrdinalFixerExamples(self): '''