From ffb5572cb2f65b05a42a71fd9366e528943d63bf Mon Sep 17 00:00:00 2001 From: Philip Bauer Date: Mon, 17 Aug 2015 15:41:21 +0200 Subject: [PATCH 1/4] test migration and custom_migration plus pep8 Conflicts: plone/app/contenttypes/migration/field_migrators.py --- .../migration/custom_migration.py | 4 + .../contenttypes/migration/field_migrators.py | 2 +- plone/app/contenttypes/testing.py | 5 + .../app/contenttypes/tests/test_migration.py | 123 ++++++++++++++++++ .../tests/test_migration_custom.py | 88 ++++++++++++- 5 files changed, 219 insertions(+), 3 deletions(-) diff --git a/plone/app/contenttypes/migration/custom_migration.py b/plone/app/contenttypes/migration/custom_migration.py index b26680b65..f3ae23b75 100644 --- a/plone/app/contenttypes/migration/custom_migration.py +++ b/plone/app/contenttypes/migration/custom_migration.py @@ -238,8 +238,12 @@ def migrate(self, dry_run=False): safe_at = at_typename.replace('_space_', '') dx_key = 'dx_%s__for__%s' % (safe_dx, safe_at) for at_field in form[at_typename]: + if form.get(dx_key) is None: + # No field-mappings + continue dx_field = form[dx_key][form[at_typename].index(at_field)] if not dx_field: + # Do not migrate field continue at_field_name, at_field_type = at_field.split('__type__') dx_field_name, dx_field_type = dx_field.split('__type__') diff --git a/plone/app/contenttypes/migration/field_migrators.py b/plone/app/contenttypes/migration/field_migrators.py index ede3edef2..1600a58b9 100644 --- a/plone/app/contenttypes/migration/field_migrators.py +++ b/plone/app/contenttypes/migration/field_migrators.py @@ -96,7 +96,7 @@ def migrate_blobimagefield(src_obj, dst_obj, src_fieldname, dst_fieldname): """ migrate an image field. Actually this field needs only to copy the existing NamedBlobImage instance - to the new dst_obj, but we do a little more in detail and create new fields + to the new dst_obj, but we do some more in detail and create new fields. """ old_image = getattr(src_obj, src_fieldname) if old_image == '': diff --git a/plone/app/contenttypes/testing.py b/plone/app/contenttypes/testing.py index 71ae5d2bd..d83248355 100644 --- a/plone/app/contenttypes/testing.py +++ b/plone/app/contenttypes/testing.py @@ -139,6 +139,11 @@ def setUpPloneSite(self, portal): bases=(PLONE_APP_CONTENTTYPES_FIXTURE,), name="PloneAppContenttypes:Functional" ) +PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_FIXTURE = PloneAppContenttypesMigration() # noqa +PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_TESTING = FunctionalTesting( + bases=(PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_FIXTURE,), + name="PloneAppContenttypes:Migration_Functional" +) PLONE_APP_CONTENTTYPES_ROBOT_TESTING = FunctionalTesting( bases=( PLONE_APP_CONTENTTYPES_FIXTURE, diff --git a/plone/app/contenttypes/tests/test_migration.py b/plone/app/contenttypes/tests/test_migration.py index dd6cc86e8..bf4c072e1 100644 --- a/plone/app/contenttypes/tests/test_migration.py +++ b/plone/app/contenttypes/tests/test_migration.py @@ -13,6 +13,7 @@ from plone.app.contenttypes.migration.utils import restore_references from plone.app.contenttypes.migration.utils import store_references from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_FUNCTIONAL_TESTING # noqa +from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_TESTING # noqa from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_MIGRATION_TESTING # noqa from plone.app.contenttypes.testing import set_browserlayer from plone.app.referenceablebehavior.referenceable import IReferenceable @@ -42,6 +43,7 @@ import json import os.path import time +import transaction import unittest2 as unittest @@ -1972,3 +1974,124 @@ def test_dxmigration_migrate_check_migration_successful_message(self): self.browser.getControl('Update').click() self.assertIn( self.good_info_message_template.format(1), self.browser.contents) + + +class MigrationFunctionalTests(unittest.TestCase): + + layer = PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_TESTING + + def setUp(self): + app = self.layer['app'] + self.portal = self.layer['portal'] + self.request = self.layer['request'] + self.request['ACTUAL_URL'] = self.portal.absolute_url() + self.request['URL'] = self.portal.absolute_url() + self.catalog = getToolByName(self.portal, "portal_catalog") + self.portal.acl_users.userFolderAddUser('admin', + 'secret', + ['Manager'], + []) + login(self.portal, 'admin') + self.portal.portal_workflow.setDefaultChain( + "simple_publication_workflow") + self.portal_url = self.portal.absolute_url() + + self.browser = Browser(app) + self.browser.handleErrors = False + self.browser.addHeader( + 'Authorization', + 'Basic %s:%s' % (SITE_OWNER_NAME, SITE_OWNER_PASSWORD,) + ) + + def tearDown(self): + try: + applyProfile(self.portal, 'plone.app.contenttypes:uninstall') + except KeyError: + pass + + def test_pac_installer_cancel(self): + qi = self.portal.portal_quickinstaller + portal_types = self.portal.portal_types + self.browser.open('%s/@@pac_installer' % self.portal_url) + self.assertFalse(qi.isProductInstalled('plone.app.contenttypes')) + self.browser.getControl('Cancel').click() + self.assertFalse(IDexterityFTI.providedBy(portal_types['Document'])) + self.assertFalse(qi.isProductInstalled('plone.app.contenttypes')) + self.assertEqual(self.browser.url, self.portal_url) + + def test_pac_installer_without_content(self): + qi = self.portal.portal_quickinstaller + portal_types = self.portal.portal_types + self.browser.open('%s/@@pac_installer' % self.portal_url) + self.assertFalse(qi.isProductInstalled('plone.app.contenttypes')) + self.assertFalse(IDexterityFTI.providedBy(portal_types['Document'])) + self.assertIn('proceed to the migration-form?', self.browser.contents) + self.browser.getControl('Install').click() + self.assertTrue(IDexterityFTI.providedBy(portal_types['Document'])) + self.assertTrue(IDexterityFTI.providedBy(portal_types['News Item'])) + self.assertTrue(qi.isProductInstalled('plone.app.contenttypes')) + self.assertIn('Migration control panel', self.browser.contents) + self.assertIn('No content to migrate.', self.browser.contents) + + def test_pac_installer_with_content(self): + # add some at content: + self.portal.invokeFactory('Document', 'doc1') + transaction.commit() + qi = self.portal.portal_quickinstaller + portal_types = self.portal.portal_types + self.browser.open('%s/@@pac_installer' % self.portal_url) + self.assertFalse(IDexterityFTI.providedBy(portal_types['Document'])) + self.assertFalse(qi.isProductInstalled('plone.app.contenttypes')) + self.assertIn('proceed to the migration-form?', self.browser.contents) + self.browser.getControl('Install').click() + self.assertFalse(IDexterityFTI.providedBy(portal_types['Document'])) + self.assertTrue(IDexterityFTI.providedBy(portal_types['News Item'])) + self.assertTrue(qi.isProductInstalled('plone.app.contenttypes')) + self.assertIn('Migration control panel', self.browser.contents) + self.assertIn('You currently have 1 archetypes objects to be migrated.', self.browser.contents) # noqa + + def test_atct_migration_form(self): + # setup session + # taken from Products.Sessions.tests.testSessionDataManager._populate + tf_name = 'temp_folder' + idmgr_name = 'browser_id_manager' + toc_name = 'temp_transient_container' + sdm_name = 'session_data_manager' + from Products.Sessions.BrowserIdManager import BrowserIdManager + from Products.Sessions.SessionDataManager import SessionDataManager + from Products.TemporaryFolder.TemporaryFolder import MountedTemporaryFolder # noqa + from Products.Transience.Transience import TransientObjectContainer + bidmgr = BrowserIdManager(idmgr_name) + tf = MountedTemporaryFolder(tf_name, title="Temporary Folder") + toc = TransientObjectContainer( + toc_name, + title='Temporary Transient Object Container', + timeout_mins=20) + session_data_manager = SessionDataManager( + id=sdm_name, + path=tf_name+'/'+toc_name, + title='Session Data Manager', + requestName='TESTOFSESSION') + self.portal._setObject(idmgr_name, bidmgr) + self.portal._setObject(sdm_name, session_data_manager) + self.portal._setObject(tf_name, tf) + transaction.commit() + self.portal.temp_folder._setObject(toc_name, toc) + + # add some at content: + self.portal.invokeFactory('Document', 'doc1') + transaction.commit() + qi = self.portal.portal_quickinstaller + portal_types = self.portal.portal_types + from zExceptions import NotFound + self.assertRaises(NotFound, self.browser.open, '%s/@@atct_migrator' % self.portal_url) # noqa + self.browser.open('%s/@@pac_installer' % self.portal_url) + self.browser.getControl('Install').click() + self.assertIn('You currently have 1 archetypes objects to be migrated.', self.browser.contents) # noqa + + self.browser.getControl(name='form.widgets.content_types:list').value = ['Document'] # noqa + self.assertEqual(self.browser.getControl(name='form.widgets.migrate_references:list').value, ['selected']) # noqa + self.browser.getControl(name='form.buttons.migrate').click() + self.assertIn('Congratulations! You migrated from Archetypes to Dexterity.', self.browser.contents) # noqa + msg = "ATDocument\n Document\n 1" + self.assertIn(msg, self.browser.contents) diff --git a/plone/app/contenttypes/tests/test_migration_custom.py b/plone/app/contenttypes/tests/test_migration_custom.py index 905277193..b56c767ce 100644 --- a/plone/app/contenttypes/tests/test_migration_custom.py +++ b/plone/app/contenttypes/tests/test_migration_custom.py @@ -1,4 +1,5 @@ # -*- coding: utf-8 -*- +from Products.CMFCore.utils import getToolByName from Products.CMFPlone.utils import safe_unicode from datetime import datetime from plone.app.contenttypes.migration.field_migrators import migrate_filefield @@ -6,14 +7,20 @@ from plone.app.contenttypes.migration.field_migrators import \ migrate_simplefield from plone.app.contenttypes.migration.utils import installTypeIfNeeded -from plone.app.contenttypes.testing import \ - PLONE_APP_CONTENTTYPES_MIGRATION_TESTING +from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_TESTING # noqa +from plone.app.contenttypes.testing import PLONE_APP_CONTENTTYPES_MIGRATION_TESTING # noqa +from plone.app.testing import SITE_OWNER_NAME +from plone.app.testing import SITE_OWNER_PASSWORD from plone.app.testing import TEST_USER_ID from plone.app.testing import applyProfile +from plone.app.testing import applyProfile +from plone.app.testing import login from plone.app.testing import setRoles +from plone.testing.z2 import Browser import pytz import os.path +import transaction import unittest2 as unittest @@ -395,3 +402,80 @@ def test_migrate_atevent_to_dxevent(self): os.environ['TZ'] = oldTZ else: del os.environ['TZ'] + + +class CustomMigrationFunctionalTests(unittest.TestCase): + + layer = PLONE_APP_CONTENTTYPES_MIGRATION_FUNCTIONAL_TESTING + + def setUp(self): + app = self.layer['app'] + self.portal = self.layer['portal'] + self.request = self.layer['request'] + self.request['ACTUAL_URL'] = self.portal.absolute_url() + self.request['URL'] = self.portal.absolute_url() + self.catalog = getToolByName(self.portal, "portal_catalog") + self.portal.acl_users.userFolderAddUser('admin', + 'secret', + ['Manager'], + []) + login(self.portal, 'admin') + self.portal.portal_workflow.setDefaultChain( + "simple_publication_workflow") + self.portal_url = self.portal.absolute_url() + + self.browser = Browser(app) + self.browser.handleErrors = False + self.browser.addHeader( + 'Authorization', + 'Basic %s:%s' % (SITE_OWNER_NAME, SITE_OWNER_PASSWORD,) + ) + + def tearDown(self): + try: + applyProfile(self.portal, 'plone.app.contenttypes:uninstall') + except KeyError: + pass + + def test_custom_migration_form(self): + """Basic test for the custom_migration form. + Field-mapping only works with javascript enabled so we migrate + only the content but not the fields. + """ + # add some at content + self.portal.invokeFactory('Document', 'doc1') + self.portal.invokeFactory('Event', 'event1') + self.portal.event1.setTitle(u'Ein Törmin') + self.portal.event1.setDescription(u'Wänn?') + self.portal.doc1.setTitle(u'Ein Döcument') + self.portal.doc1.setDescription(u'Sö was') + transaction.commit() + self.browser.open('%s/@@pac_installer' % self.portal_url) + self.browser.getControl('Install').click() + # open custom-migration-form + self.browser.open('%s/@@custom_migration' % self.portal_url) + results = self.browser.contents + self.assertIn('Custom types migration control panel', results) + self.assertIn('', results) # noqa + self.assertEqual(self.browser.getControl(name='dx_select_Document').value, ['']) # noqa + # chose to migrate to Link + self.browser.getControl(name='dx_select_Document').value = ['Link'] + self.assertIn('', results) # noqa + # chose to migrate to Link + self.browser.getControl(name='dx_select_Event').value = ['Link'] + # run migration + self.browser.getControl(name='form.button.Migrate').click() + results = self.browser.contents + self.assertIn('Migration applied succesfully for 1 "Document" items.', results) # noqa + self.assertIn('Migration applied succesfully for 1 "Event" items.', results) # noqa + self.assertIn('No content to migrate.', results) + link1 = self.portal['doc1'] + self.assertEqual(link1.portal_type, 'Link') + self.assertEqual(link1.title, u'Ein D\xf6cument') + self.assertEqual(link1.description, u'S\xf6 was') + self.assertEqual(self.portal['event1'].portal_type, 'Link') + # we did not migrate the fields so lets find out if it is a real Link + link1.remote_url = 'http://www.starzel.de' + view = link1() + self.assertIn(u'

Ein D\xf6cument

', view) # noqa + self.assertIn(u'The link address is:\n http://www.starzel.de', view) # noqa From 30f9b7ea7eeee40c695f174a5490b2981bcfe032 Mon Sep 17 00:00:00 2001 From: Philip Bauer Date: Thu, 15 Oct 2015 16:25:11 +0300 Subject: [PATCH 2/4] Fix custom migration from and to types with spaces in the type-name. --- docs/CHANGES.rst | 28 ++++++++++++ .../migration/custom_migration.py | 19 ++++---- plone/app/contenttypes/migration/migration.py | 20 ++++++--- .../tests/test_migration_custom.py | 43 +++++++++++++++++++ 4 files changed, 95 insertions(+), 15 deletions(-) diff --git a/docs/CHANGES.rst b/docs/CHANGES.rst index c96456274..d1b2a078e 100644 --- a/docs/CHANGES.rst +++ b/docs/CHANGES.rst @@ -10,6 +10,34 @@ Changelog 1.1b6 (2015-10-17) ------------------ +- Fix custom migration from and to types with spaces in the type-name. + [pbauer] + +- Fix full_view when content is not IUUIDAware (like the portal). + Fixes https://github.com/plone/Products.CMFPlone/issues/1109. + [pbauer] + +- Add plone.app.linkintegrity to dependencies due to test-issues. + [pbauer] + + +1.2.4 (2015-09-27) +------------------ + +- fix full_view error when collection contains itself + [vangheem] + +- test_content_profile: do not appy Products.CMFPlone:plone. + [maurits] + + +1.2.3 (2015-09-20) +------------------ + +- Do not raise an exception for items where @@full_view_item throws an + exception. Instead hide the object. + [pbauer] + - Do not raise errors when IPrimaryFieldInfo(obj) fails (e.g. when the Schema-Cache is gone). Fixes https://github.com/plone/Products.CMFPlone/issues/839 diff --git a/plone/app/contenttypes/migration/custom_migration.py b/plone/app/contenttypes/migration/custom_migration.py index f3ae23b75..ad2f6d89f 100644 --- a/plone/app/contenttypes/migration/custom_migration.py +++ b/plone/app/contenttypes/migration/custom_migration.py @@ -226,22 +226,25 @@ def migrate(self, dry_run=False): if not form[k] or (dry_run and k != form.get('tested_type')): # nothing selected in this select, continue continue - at_typename = k[10:] - dx_typename = form[k] + form_at_typename = k[10:] + form_dx_typename = form[k] + at_typename = form_at_typename.replace('_space_', ' ') + dx_typename = form_dx_typename.replace('_space_', ' ') + data[at_typename] = {'target_type': dx_typename, 'field_mapping': []} # now handle fields mapping for found DX/AT type migration - # definition we have 2 keys we relevant mappings, first key + # definition we have 2 keys with relevant mappings, first key # is the AT typename second key is a particular key like # 'dx_DXPortalType__for__MyATPortalType - safe_dx = dx_typename.replace('_space_', '') - safe_at = at_typename.replace('_space_', '') - dx_key = 'dx_%s__for__%s' % (safe_dx, safe_at) - for at_field in form[at_typename]: + dx_key = 'dx_%s__for__%s' % (form_dx_typename, + form_at_typename) + for at_field in form[form_at_typename]: if form.get(dx_key) is None: # No field-mappings continue - dx_field = form[dx_key][form[at_typename].index(at_field)] + dx_field = form[dx_key][form[form_at_typename].index( + at_field)] if not dx_field: # Do not migrate field continue diff --git a/plone/app/contenttypes/migration/migration.py b/plone/app/contenttypes/migration/migration.py index a0a846de7..446fc673f 100644 --- a/plone/app/contenttypes/migration/migration.py +++ b/plone/app/contenttypes/migration/migration.py @@ -467,13 +467,19 @@ def migrateCustomAT(fields_mapping, src_type, dst_type, dry_run=False): if fti is None or IDexterityFTI.providedBy(fti): # Get the needed info from an instance of the type catalog = portal.portal_catalog - brain = catalog(portal_type=src_type, sort_limit=1)[0] - src_obj = brain.getObject() - if IDexterityContent.providedBy(src_obj): - logger.error( - '%s should not be dexterity object!' % src_obj.absolute_url()) - is_folderish = getattr(src_obj, 'isPrincipiaFolderish', False) - src_meta_type = src_obj.meta_type + import pdb; pdb.set_trace() + brains = catalog(portal_type=src_type, sort_limit=1) + if not brains: + # no item? assume stuff + is_folderish = False + src_meta_type = src_type + else: + src_obj = brains[0].getObject() + if IDexterityContent.providedBy(src_obj): + logger.error( + '%s should not be dexterity object!' % src_obj.absolute_url()) + is_folderish = getattr(src_obj, 'isPrincipiaFolderish', False) + src_meta_type = src_obj.meta_type else: # Get info from at-fti src_meta_type = fti.content_meta_type diff --git a/plone/app/contenttypes/tests/test_migration_custom.py b/plone/app/contenttypes/tests/test_migration_custom.py index b56c767ce..05fed4cd3 100644 --- a/plone/app/contenttypes/tests/test_migration_custom.py +++ b/plone/app/contenttypes/tests/test_migration_custom.py @@ -479,3 +479,46 @@ def test_custom_migration_form(self): view = link1() self.assertIn(u'

Ein D\xf6cument

', view) # noqa self.assertIn(u'The link address is:\n http://www.starzel.de', view) # noqa + + def test_custom_migration_form_for_types_with_spaces(self): + """Basic test for the custom_migration form. + Field-mapping only works with javascript enabled so we migrate + only the content but not the fields. + """ + # add some at content + self.portal.invokeFactory('News Item', 'news1') + self.portal.invokeFactory('Event', 'event1') + self.portal.event1.setTitle(u'Ein Törmin') + self.portal.event1.setDescription(u'Wänn?') + self.portal.news1.setTitle(u'Ein News Item') + self.portal.news1.setDescription(u'Sö was') + transaction.commit() + self.browser.open('%s/@@pac_installer' % self.portal_url) + self.browser.getControl('Install').click() + # open custom-migration-form + self.browser.open('%s/@@custom_migration' % self.portal_url) + results = self.browser.contents + self.assertIn('Custom types migration control panel', results) + self.assertIn('', results) # noqa + self.assertEqual(self.browser.getControl(name='dx_select_News_space_Item').value, ['']) # noqa + # chose to migrate to Link + self.browser.getControl(name='dx_select_News_space_Item').value = ['Link'] # noqa + self.assertIn('', results) # noqa + # chose to migrate to Link + self.browser.getControl(name='dx_select_Event').value = ['Link'] + # run migration + self.browser.getControl(name='form.button.Migrate').click() + results = self.browser.contents + self.assertIn('Migration applied succesfully for 1 "News Item" items.', results) # noqa + self.assertIn('Migration applied succesfully for 1 "Event" items.', results) # noqa + self.assertIn('No content to migrate.', results) + link1 = self.portal['news1'] + self.assertEqual(link1.portal_type, 'Link') + self.assertEqual(link1.title, u'Ein News Item') + self.assertEqual(link1.description, u'S\xf6 was') + self.assertEqual(self.portal['event1'].portal_type, 'Link') + # we did not migrate the fields so lets find out if it is a real Link + link1.remote_url = 'http://www.starzel.de' + view = link1() + self.assertIn(u'

Ein News Item

', view) # noqa + self.assertIn(u'The link address is:\n http://www.starzel.de', view) # noqa From 170604e7dd91f362d0a6a1433be751717fe09222 Mon Sep 17 00:00:00 2001 From: Philip Bauer Date: Thu, 15 Oct 2015 16:47:22 +0300 Subject: [PATCH 3/4] remove pdb --- plone/app/contenttypes/migration/migration.py | 1 - 1 file changed, 1 deletion(-) diff --git a/plone/app/contenttypes/migration/migration.py b/plone/app/contenttypes/migration/migration.py index 446fc673f..d441c1b1c 100644 --- a/plone/app/contenttypes/migration/migration.py +++ b/plone/app/contenttypes/migration/migration.py @@ -467,7 +467,6 @@ def migrateCustomAT(fields_mapping, src_type, dst_type, dry_run=False): if fti is None or IDexterityFTI.providedBy(fti): # Get the needed info from an instance of the type catalog = portal.portal_catalog - import pdb; pdb.set_trace() brains = catalog(portal_type=src_type, sort_limit=1) if not brains: # no item? assume stuff From 6bc6c4f2c1217da910d0fa55ed20a32d223dc908 Mon Sep 17 00:00:00 2001 From: Campbell Date: Sun, 8 Nov 2015 15:03:21 +0200 Subject: [PATCH 4/4] Fix bad merge of CHANGES.rst --- docs/CHANGES.rst | 7 ------- 1 file changed, 7 deletions(-) diff --git a/docs/CHANGES.rst b/docs/CHANGES.rst index d1b2a078e..358ef8a32 100644 --- a/docs/CHANGES.rst +++ b/docs/CHANGES.rst @@ -13,13 +13,6 @@ Changelog - Fix custom migration from and to types with spaces in the type-name. [pbauer] -- Fix full_view when content is not IUUIDAware (like the portal). - Fixes https://github.com/plone/Products.CMFPlone/issues/1109. - [pbauer] - -- Add plone.app.linkintegrity to dependencies due to test-issues. - [pbauer] - 1.2.4 (2015-09-27) ------------------