From 0a518111da12fd947163d1a820b12eb44de8a5b9 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Wed, 15 Feb 2023 17:34:30 -0800 Subject: [PATCH 01/31] Allow Django 4.2 (#227) * Allow Django 4.2 * allow Django 4.2 --- azure-pipelines.yml | 20 ++++++++++++++++++++ setup.py | 3 ++- tox.ini | 8 +++++--- 3 files changed, 27 insertions(+), 4 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index 3436990c..386ea5e7 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -24,6 +24,16 @@ jobs: strategy: matrix: + Python3.10 - Django 4.2: + python.version: '3.10' + tox.env: 'py310-django42' + Python 3.9 - Django 4.2: + python.version: '3.9' + tox.env: 'py39-django42' + Python 3.8 - Django 4.2: + python.version: '3.8' + tox.env: 'py38-django42' + Python3.10 - Django 4.1: python.version: '3.10' tox.env: 'py310-django41' @@ -111,6 +121,16 @@ jobs: strategy: matrix: + Python3.10 - Django 4.2: + python.version: '3.10' + tox.env: 'py310-django42' + Python 3.9 - Django 4.2: + python.version: '3.9' + tox.env: 'py39-django42' + Python 3.8 - Django 4.2: + python.version: '3.8' + tox.env: 'py38-django42' + Python3.10 - Django 4.1: python.version: '3.10' tox.env: 'py310-django41' diff --git a/setup.py b/setup.py index eba21613..f632e363 100644 --- a/setup.py +++ b/setup.py @@ -18,6 +18,7 @@ 'Framework :: Django :: 3.2', 'Framework :: Django :: 4.0', 'Framework :: Django :: 4.1', + 'Framework :: Django :: 4.2', ] this_directory = path.abspath(path.dirname(__file__)) @@ -39,7 +40,7 @@ license='BSD', packages=find_packages(), install_requires=[ - 'django>=2.2,<4.2', + 'django>=2.2,<4.3', 'pyodbc>=3.0', 'pytz', ], diff --git a/tox.ini b/tox.ini index 12d89b3c..91db1c4a 100644 --- a/tox.ini +++ b/tox.ini @@ -1,9 +1,10 @@ [tox] envlist = {py36,py37,py38,py39}-django32, - {py38, py39, py310}-django40 - {py38, py39, py310}-django41 - + {py38, py39, py310}-django40, + {py38, py39, py310}-django41, + {py38, py39, py310}-django42 + [testenv] allowlist_externals = bash @@ -19,3 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 + django42: django>=4.2a1,<4.3 \ No newline at end of file From 81018de54c7669ba64d2ddac9927f3476da9d6db Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Tue, 21 Feb 2023 12:22:00 -0800 Subject: [PATCH 02/31] Fix errors with raising FullResultSet exception and with alter_column_type_sql() and collate_sql() functions (#229) * fix error with raising fullresultset * add django4.2 condition * fix alter_column_type_sql and collate_sql to take 2 additional arguments * delete argument 'old_rel_collation' * fix arguments names --- mssql/compiler.py | 16 ++++++++++++++-- mssql/schema.py | 37 ++++++++++++++++++++++++++++--------- 2 files changed, 42 insertions(+), 11 deletions(-) diff --git a/mssql/compiler.py b/mssql/compiler.py index e71aa14f..91ff9bc1 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -15,6 +15,8 @@ from django.db.utils import NotSupportedError if django.VERSION >= (3, 1): from django.db.models.fields.json import compile_json_path, KeyTransform as json_KeyTransform +if django.VERSION >= (4, 2): + from django.core.exceptions import FullResultSet def _as_sql_agv(self, compiler, connection): return self.as_sql(compiler, connection, template='%(function)s(CONVERT(float, %(field)s))') @@ -233,8 +235,18 @@ def as_sql(self, with_limits=True, with_col_aliases=False): # This must come after 'select', 'ordering', and 'distinct' -- see # docstring of get_from_clause() for details. from_, f_params = self.get_from_clause() - where, w_params = self.compile(self.where) if self.where is not None else ("", []) - having, h_params = self.compile(self.having) if self.having is not None else ("", []) + if django.VERSION >= (4, 2): + try: + where, w_params = self.compile(self.where) if self.where is not None else ("", []) + except FullResultSet: + where, w_params = "", [] + try: + having, h_params = self.compile(self.having) if self.having is not None else ("", []) + except FullResultSet: + having, h_params = "", [] + else: + where, w_params = self.compile(self.where) if self.where is not None else ("", []) + having, h_params = self.compile(self.having) if self.having is not None else ("", []) params = [] result = ['SELECT'] diff --git a/mssql/schema.py b/mssql/schema.py index f977fdc0..00ee5403 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -161,9 +161,14 @@ def _alter_column_null_sql(self, model, old_field, new_field): [], ) - def _alter_column_type_sql(self, model, old_field, new_field, new_type): - new_type = self._set_field_new_type_null_status(old_field, new_type) - return super()._alter_column_type_sql(model, old_field, new_field, new_type) + if django_version >= (4, 2): + def _alter_column_type_sql(self, model, old_field, new_field, new_type, old_collation, new_collation): + new_type = self._set_field_new_type_null_status(old_field, new_type) + return super()._alter_column_type_sql(model, old_field, new_field, new_type, old_collation, new_collation) + else: + def _alter_column_type_sql(self, model, old_field, new_field, new_type): + new_type = self._set_field_new_type_null_status(old_field, new_type) + return super()._alter_column_type_sql(model, old_field, new_field, new_type) def alter_unique_together(self, model, old_unique_together, new_unique_together): """ @@ -443,7 +448,12 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, post_actions = [] # Type change? if old_type != new_type: - fragment, other_actions = self._alter_column_type_sql(model, old_field, new_field, new_type) + if django_version >= (4, 2): + fragment, other_actions = self._alter_column_type_sql( + model, old_field, new_field, new_type, old_collation=None, new_collation=None + ) + else: + fragment, other_actions = self._alter_column_type_sql(model, old_field, new_field, new_type) actions.append(fragment) post_actions.extend(other_actions) # Drop unique constraint, SQL Server requires explicit deletion @@ -683,9 +693,14 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, for old_rel, new_rel in rels_to_update: rel_db_params = new_rel.field.db_parameters(connection=self.connection) rel_type = rel_db_params['type'] - fragment, other_actions = self._alter_column_type_sql( - new_rel.related_model, old_rel.field, new_rel.field, rel_type - ) + if django_version >= (4, 2): + fragment, other_actions = self._alter_column_type_sql( + new_rel.related_model, old_rel.field, new_rel.field, rel_type, old_collation=None, new_collation=None + ) + else: + fragment, other_actions = self._alter_column_type_sql( + new_rel.related_model, old_rel.field, new_rel.field, rel_type + ) # Drop related_model indexes, so it can be altered index_names = self._db_table_constraint_names(old_rel.related_model._meta.db_table, index=True) for index_name in index_names: @@ -1262,8 +1277,12 @@ def add_constraint(self, model, constraint): (constraint.condition.connector, constraint.name)) super().add_constraint(model, constraint) - def _collate_sql(self, collation): - return ' COLLATE ' + collation + if django_version >= (4, 2): + def _collate_sql(self, collation, old_collation=None, table_name=None): + return ' COLLATE ' + collation if collation else "" + else: + def _collate_sql(self, collation): + return ' COLLATE ' + collation def _create_index_name(self, table_name, column_names, suffix=""): index_name = super()._create_index_name(table_name, column_names, suffix) From 06e07885ac5ae9a0b6ac9333ad7a8d37ed302360 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Fri, 3 Mar 2023 10:01:04 -0800 Subject: [PATCH 03/31] fix last_executed_query() to properly replace placeholders with params (#234) --- mssql/operations.py | 8 +++++++- testapp/settings.py | 1 - 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/mssql/operations.py b/mssql/operations.py index 9222ea5e..fcc0608e 100644 --- a/mssql/operations.py +++ b/mssql/operations.py @@ -418,7 +418,13 @@ def last_executed_query(self, cursor, sql, params): exists for database backends to provide a better implementation according to their own quoting schemes. """ - return super().last_executed_query(cursor, cursor.last_sql, cursor.last_params) + if params: + if isinstance(params, list): + params = tuple(params) + return sql % params + # Just return sql when there are no parameters. + else: + return sql def savepoint_create_sql(self, sid): """ diff --git a/testapp/settings.py b/testapp/settings.py index 767f0f3d..311e9175 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -152,7 +152,6 @@ 'schema.tests.SchemaTests.test_unique_and_reverse_m2m', 'schema.tests.SchemaTests.test_unique_no_unnecessary_fk_drops', 'select_for_update.tests.SelectForUpdateTests.test_for_update_after_from', - 'backends.tests.LastExecutedQueryTest.test_last_executed_query', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_exact_lookup', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_greaterthan_lookup', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_year_lessthan_lookup', From 68751515146721466fd73b25ec9239af1ac27aca Mon Sep 17 00:00:00 2001 From: mShan0 Date: Fri, 10 Mar 2023 11:12:56 -0800 Subject: [PATCH 04/31] disable allows_group_by_select_index --- mssql/features.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/mssql/features.py b/mssql/features.py index a60a9283..faaa0bbb 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -6,6 +6,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): + allows_group_by_select_index = False allow_sliced_subqueries_with_in = False can_introspect_autofield = True can_introspect_json_field = False @@ -71,4 +72,4 @@ def introspected_field_types(self): return { **super().introspected_field_types, "DurationField": "BigIntegerField", - } \ No newline at end of file + } From 9ac7c2f0c9bee278016d3973d3dc50b072397100 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Fri, 10 Mar 2023 11:33:37 -0800 Subject: [PATCH 05/31] unskip old tests --- testapp/settings.py | 8 -------- 1 file changed, 8 deletions(-) diff --git a/testapp/settings.py b/testapp/settings.py index 311e9175..ee5ee11d 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -99,11 +99,8 @@ TEST_RUNNER = "testapp.runners.ExcludedTestSuiteRunner" EXCLUDED_TESTS = [ - 'aggregation.tests.AggregateTestCase.test_expression_on_aggregation', - 'aggregation_regress.tests.AggregationTests.test_annotated_conditional_aggregate', 'aggregation_regress.tests.AggregationTests.test_annotation_with_value', 'aggregation.tests.AggregateTestCase.test_distinct_on_aggregate', - 'annotations.tests.NonAggregateAnnotationTestCase.test_annotate_exists', 'custom_lookups.tests.BilateralTransformTests.test_transform_order_by', 'expressions.tests.BasicExpressionsTests.test_filtering_on_annotate_that_uses_q', 'expressions.tests.BasicExpressionsTests.test_order_by_exists', @@ -165,17 +162,13 @@ 'backends.tests.BackendTestCase.test_queries', 'introspection.tests.IntrospectionTests.test_smallautofield', 'schema.tests.SchemaTests.test_inline_fk', - 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_exists', - 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values_collision', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone', 'expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction', 'expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction', 'expressions.tests.FTimeDeltaTests.test_time_subquery_subtraction', 'migrations.test_operations.OperationTests.test_alter_field_reloads_state_on_fk_with_to_field_target_type_change', 'schema.tests.SchemaTests.test_alter_smallint_pk_to_smallautofield_pk', - 'annotations.tests.NonAggregateAnnotationTestCase.test_combined_expression_annotation_with_aggregation', - 'db_functions.comparison.test_cast.CastTests.test_cast_to_integer', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_func', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_iso_weekday_func', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func', @@ -271,7 +264,6 @@ # Django 4.1 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_on_exists', 'aggregation.tests.AggregateTestCase.test_aggregation_exists_multivalued_outeref', - 'annotations.tests.NonAggregateAnnotationTestCase.test_full_expression_annotation_with_aggregation', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_lookup_name_sql_injection', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_lookup_name_sql_injection', 'schema.tests.SchemaTests.test_autofield_to_o2o', From 9816cdfe212c853a922c506dc938b05f3b2ac1b4 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Mon, 13 Mar 2023 11:07:55 -0700 Subject: [PATCH 06/31] unskip some tests --- testapp/settings.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/testapp/settings.py b/testapp/settings.py index ee5ee11d..d87fc904 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -101,6 +101,7 @@ EXCLUDED_TESTS = [ 'aggregation_regress.tests.AggregationTests.test_annotation_with_value', 'aggregation.tests.AggregateTestCase.test_distinct_on_aggregate', + 'annotations.tests.NonAggregateAnnotationTestCase.test_annotate_exists', 'custom_lookups.tests.BilateralTransformTests.test_transform_order_by', 'expressions.tests.BasicExpressionsTests.test_filtering_on_annotate_that_uses_q', 'expressions.tests.BasicExpressionsTests.test_order_by_exists', @@ -162,6 +163,8 @@ 'backends.tests.BackendTestCase.test_queries', 'introspection.tests.IntrospectionTests.test_smallautofield', 'schema.tests.SchemaTests.test_inline_fk', + 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_exists', + 'aggregation.tests.AggregateTestCase.test_aggregation_subquery_annotation_values_collision', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_func_with_timezone', 'expressions.tests.FTimeDeltaTests.test_date_subquery_subtraction', 'expressions.tests.FTimeDeltaTests.test_datetime_subquery_subtraction', From 8e5c9cdac6918df92638d4f1bcfec62cd5cfb090 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Mon, 13 Mar 2023 12:16:39 -0700 Subject: [PATCH 07/31] skip more tests --- testapp/settings.py | 1 + 1 file changed, 1 insertion(+) diff --git a/testapp/settings.py b/testapp/settings.py index d87fc904..3fb637bb 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -267,6 +267,7 @@ # Django 4.1 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_on_exists', 'aggregation.tests.AggregateTestCase.test_aggregation_exists_multivalued_outeref', + 'annotations.tests.NonAggregateAnnotationTestCase.test_full_expression_annotation_with_aggregation', 'db_functions.datetime.test_extract_trunc.DateFunctionWithTimeZoneTests.test_extract_lookup_name_sql_injection', 'db_functions.datetime.test_extract_trunc.DateFunctionTests.test_extract_lookup_name_sql_injection', 'schema.tests.SchemaTests.test_autofield_to_o2o', From 303859c26cb40ebb116cd136a4a8c196d4f539a3 Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Wed, 15 Mar 2023 13:54:35 -0700 Subject: [PATCH 08/31] Use latest Django 4.2 beta for tox (#238) --- tox.ini | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tox.ini b/tox.ini index 91db1c4a..8737f9c0 100644 --- a/tox.ini +++ b/tox.ini @@ -4,7 +4,7 @@ envlist = {py38, py39, py310}-django40, {py38, py39, py310}-django41, {py38, py39, py310}-django42 - + [testenv] allowlist_externals = bash @@ -20,4 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 - django42: django>=4.2a1,<4.3 \ No newline at end of file + django42: django>=4.2b1,<4.3 From e196f5a02f1867fb51b7789acbb9a157df8ebfc0 Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Mon, 20 Mar 2023 15:51:30 -0700 Subject: [PATCH 09/31] use 4.2 rc1 branch (#240) --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 8737f9c0..3376ddca 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 - django42: django>=4.2b1,<4.3 + django42: django>=4.2rc1,<4.3 From 5a630ba52ccc26947129cbeee398b7a7c8aae7cc Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Thu, 23 Mar 2023 16:45:56 -0700 Subject: [PATCH 10/31] allow partial support for filtering against window functions (#239) --- mssql/compiler.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/mssql/compiler.py b/mssql/compiler.py index 91ff9bc1..2dfe1b6c 100644 --- a/mssql/compiler.py +++ b/mssql/compiler.py @@ -230,6 +230,9 @@ def as_sql(self, with_limits=True, with_col_aliases=False): if not getattr(features, 'supports_select_{}'.format(combinator)): raise NotSupportedError('{} is not supported on this database backend.'.format(combinator)) result, params = self.get_combinator_sql(combinator, self.query.combinator_all) + elif django.VERSION >= (4, 2) and self.qualify: + result, params = self.get_qualify_sql() + order_by = None else: distinct_fields, distinct_params = self.get_distinct() # This must come after 'select', 'ordering', and 'distinct' -- see From adc1409feb09f63bdf1e2b5a5e17c9c2ff9b7403 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Fri, 24 Mar 2023 10:10:08 -0700 Subject: [PATCH 11/31] add subsecond support to Now() (#242) --- mssql/functions.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/mssql/functions.py b/mssql/functions.py index ddf34ca7..dd7a1d3d 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -10,6 +10,7 @@ from django.db.models.expressions import Case, Exists, Expression, OrderBy, When, Window from django.db.models.fields import BinaryField, Field from django.db.models.functions import Cast, NthValue, MD5, SHA1, SHA224, SHA256, SHA384, SHA512 +from django.db.models.functions.datetime import Now from django.db.models.functions.math import ATan2, Ln, Log, Mod, Round, Degrees, Radians, Power from django.db.models.functions.text import Replace from django.db.models.lookups import In, Lookup @@ -123,6 +124,10 @@ def sqlserver_exists(self, compiler, connection, template=None, **extra_context) sql = 'CASE WHEN {} THEN 1 ELSE 0 END'.format(sql) return sql, params +def sqlserver_now(self, compiler, connection, **extra_context): + return self.as_sql( + compiler, connection, template="SYSDATETIME()", **extra_context + ) def sqlserver_lookup(self, compiler, connection): # MSSQL doesn't allow EXISTS() to be compared to another expression @@ -456,6 +461,7 @@ def sqlserver_sha512(self, compiler, connection, **extra_context): Round.as_microsoft = sqlserver_round Window.as_microsoft = sqlserver_window Replace.as_microsoft = sqlserver_replace +Now.as_microsoft = sqlserver_now MD5.as_microsoft = sqlserver_md5 SHA1.as_microsoft = sqlserver_sha1 SHA224.as_microsoft = sqlserver_sha224 From 0c4d052555d958ef17ee8278be40a74853acfb2c Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Tue, 4 Apr 2023 10:15:53 -0700 Subject: [PATCH 12/31] assign value to display_size (#244) --- mssql/introspection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index c5645f30..f23af078 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -107,7 +107,7 @@ def get_table_description(self, cursor, table_name, identity_check=True): """ # map pyodbc's cursor.columns to db-api cursor description - columns = [[c[3], c[4], None, c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] + columns = [[c[3], c[4], c[6], c[6], c[6], c[8], c[10], c[12]] for c in cursor.columns(table=table_name)] if not columns: raise DatabaseError(f"Table {table_name} does not exist.") From f9a1b43dcb61e9874b754be6ccc78260536addb6 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Tue, 4 Apr 2023 10:27:19 -0700 Subject: [PATCH 13/31] add latest django 4.2 branch to ci --- tox.ini | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tox.ini b/tox.ini index 3376ddca..7d2794ef 100644 --- a/tox.ini +++ b/tox.ini @@ -20,4 +20,4 @@ deps = django32: django==3.2.* django40: django>=4.0a1,<4.1 django41: django>=4.1a1,<4.2 - django42: django>=4.2rc1,<4.3 + django42: django>=4.2,<4.3 From c7445a89ec206491d231cb99d9b94c2404af34b8 Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Mon, 17 Apr 2023 10:59:33 -0700 Subject: [PATCH 14/31] allow comments on columns and tables --- mssql/features.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mssql/features.py b/mssql/features.py index faaa0bbb..a27e90b7 100644 --- a/mssql/features.py +++ b/mssql/features.py @@ -33,6 +33,7 @@ class DatabaseFeatures(BaseDatabaseFeatures): requires_literal_defaults = True requires_sqlparse_for_splitting = False supports_boolean_expr_in_select_clause = False + supports_comments = True supports_covering_indexes = True supports_deferrable_unique_constraints = False supports_expression_indexes = False From 4a37e3278c53724470b2423d129bfae7bf14c855 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Mon, 24 Apr 2023 12:44:42 -0700 Subject: [PATCH 15/31] raise an error when batch_size is zero. (#259) --- mssql/functions.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/functions.py b/mssql/functions.py index dd7a1d3d..4fdedc7d 100644 --- a/mssql/functions.py +++ b/mssql/functions.py @@ -292,7 +292,7 @@ def bulk_update_with_default(self, objs, fields, batch_size=None, default=0): SQL Server require that at least one of the result expressions in a CASE specification must be an expression other than the NULL constant. Patched with a default value 0. The user can also pass a custom default value for CASE statement. """ - if batch_size is not None and batch_size < 0: + if batch_size is not None and batch_size <= 0: raise ValueError('Batch size must be a positive integer.') if not fields: raise ValueError('Field names must be given to bulk_update().') From 3375f3aba1fa7d94e2468dcc6e1da1218300a371 Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Wed, 3 May 2023 15:32:02 -0700 Subject: [PATCH 16/31] replicate get or create test for mssql (#265) --- testapp/migrations/0024_publisher_book.py | 58 +++++++++++++++++++++++ testapp/models.py | 20 +++++++- testapp/settings.py | 3 ++ testapp/tests/test_getorcreate.py | 41 ++++++++++++++++ 4 files changed, 120 insertions(+), 2 deletions(-) create mode 100644 testapp/migrations/0024_publisher_book.py create mode 100644 testapp/tests/test_getorcreate.py diff --git a/testapp/migrations/0024_publisher_book.py b/testapp/migrations/0024_publisher_book.py new file mode 100644 index 00000000..b555d0d0 --- /dev/null +++ b/testapp/migrations/0024_publisher_book.py @@ -0,0 +1,58 @@ +# Generated by Django 4.2 on 2023-05-03 15:08 + +from django.db import migrations, models +import django.db.models.deletion + + +class Migration(migrations.Migration): + + dependencies = [ + ("testapp", "0023_number"), + ] + + operations = [ + migrations.CreateModel( + name="Publisher", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=100)), + ], + ), + migrations.CreateModel( + name="Book", + fields=[ + ( + "id", + models.AutoField( + auto_created=True, + primary_key=True, + serialize=False, + verbose_name="ID", + ), + ), + ("name", models.CharField(max_length=100)), + ("updated", models.DateTimeField(auto_now=True)), + ( + "authors", + models.ManyToManyField(related_name="books", to="testapp.author"), + ), + ( + "publisher", + models.ForeignKey( + db_column="publisher_id_column", + on_delete=django.db.models.deletion.CASCADE, + related_name="books", + to="testapp.publisher", + ), + ), + ], + ), + ] diff --git a/testapp/models.py b/testapp/models.py index fb5fdbcf..f92d10f2 100644 --- a/testapp/models.py +++ b/testapp/models.py @@ -9,7 +9,7 @@ from django.db.models import Q from django.utils import timezone -# We are using this Mixin to test casting of BigAuto and Auto fields +# We are using this Mixin to test casting of BigAuto and Auto fields class BigAutoFieldMixin(models.Model): id = models.BigAutoField(primary_key=True) @@ -229,4 +229,20 @@ class Number(models.Model): decimal_value = models.DecimalField(max_digits=20, decimal_places=17, null=True) def __str__(self): - return "%i, %.3f, %.17f" % (self.integer, self.float, self.decimal_value) \ No newline at end of file + return "%i, %.3f, %.17f" % (self.integer, self.float, self.decimal_value) + + +class Publisher(models.Model): + name = models.CharField(max_length=100) + + +class Book(models.Model): + name = models.CharField(max_length=100) + authors = models.ManyToManyField(Author, related_name="books") + publisher = models.ForeignKey( + Publisher, + models.CASCADE, + related_name="books", + db_column="publisher_id_column", + ) + updated = models.DateTimeField(auto_now=True) diff --git a/testapp/settings.py b/testapp/settings.py index 3fb637bb..9593692d 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -289,6 +289,9 @@ 'model_fields.test_jsonfield.TestQuerying.test_lookups_with_key_transform', 'model_fields.test_jsonfield.TestQuerying.test_ordering_grouping_by_count', 'model_fields.test_jsonfield.TestQuerying.test_has_key_number', + + # Django 4.2 + 'get_or_create.tests.UpdateOrCreateTests.test_update_only_defaults_and_pre_save_fields_when_local_fields' ] REGEX_TESTS = [ diff --git a/testapp/tests/test_getorcreate.py b/testapp/tests/test_getorcreate.py new file mode 100644 index 00000000..1fe9baf0 --- /dev/null +++ b/testapp/tests/test_getorcreate.py @@ -0,0 +1,41 @@ +# Copyright (c) Microsoft Corporation. +# Licensed under the BSD license. +from unittest import skipUnless + +from django import VERSION +from django.test import TestCase +from django.db import connection +from django.test.utils import CaptureQueriesContext + +from ..models import Book, Publisher + +DJANGO42 = VERSION >= (4, 2) + +# Copied from Django test suite but modified to test our code +@skipUnless(DJANGO42, "Django 4.2 specific tests") +class UpdateOrCreateTests(TestCase): + + def test_update_only_defaults_and_pre_save_fields_when_local_fields(self): + publisher = Publisher.objects.create(name="Acme Publishing") + book = Book.objects.create(publisher=publisher, name="The Book of Ed & Fred") + + for defaults in [{"publisher": publisher}, {"publisher_id": publisher}]: + with self.subTest(defaults=defaults): + with CaptureQueriesContext(connection) as captured_queries: + book, created = Book.objects.update_or_create( + pk=book.pk, + defaults=defaults, + ) + self.assertIs(created, False) + update_sqls = [ + q["sql"] for q in captured_queries if "UPDATE" in q["sql"] + ] + self.assertEqual(len(update_sqls), 1) + update_sql = update_sqls[0] + self.assertIsNotNone(update_sql) + self.assertIn( + connection.ops.quote_name("publisher_id_column"), update_sql + ) + self.assertIn(connection.ops.quote_name("updated"), update_sql) + # Name should not be updated. + self.assertNotIn(connection.ops.quote_name("name"), update_sql) From 068c7fc201e48838845993caec91ab1edabfa926 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Tue, 23 May 2023 14:44:03 -0700 Subject: [PATCH 17/31] add table comment to `get_table_list` query --- mssql/introspection.py | 34 ++++++++++++++++++++++++++-------- 1 file changed, 26 insertions(+), 8 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index f23af078..5e0feadd 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -4,10 +4,12 @@ from django.db import DatabaseError import pyodbc as Database +from collections import namedtuple + from django import VERSION -from django.db.backends.base.introspection import ( - BaseDatabaseIntrospection, FieldInfo, TableInfo, -) +from django.db.backends.base.introspection import BaseDatabaseIntrospection +from django.db.backends.base.introspection import FieldInfo +from django.db.backends.base.introspection import TableInfo as BaseTableInfo from django.db.models.indexes import Index from django.conf import settings @@ -15,6 +17,7 @@ SQL_BIGAUTOFIELD = -777444 SQL_TIMESTAMP_WITH_TIMEZONE = -155 +TableInfo = namedtuple("TableInfo", BaseTableInfo._fields + ("comment",)) def get_schema_name(): return getattr(settings, 'SCHEMA_TO_INSPECT', 'SCHEMA_NAME()') @@ -71,13 +74,28 @@ def get_table_list(self, cursor): """ Returns a list of table and view names in the current database. """ - sql = 'SELECT TABLE_NAME, TABLE_TYPE FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_SCHEMA = %s' % ( + sql = """SELECT + TABLE_NAME, + TABLE_TYPE, + CAST(ep.value AS VARCHAR) AS COMMENT + FROM INFORMATION_SCHEMA.TABLES i + INNER JOIN sys.tables t ON t.name = i.TABLE_NAME + LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id + WHERE + ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id = 0) OR ep.value IS NULL) + AND + i.TABLE_SCHEMA = %s""" % ( get_schema_name()) cursor.execute(sql) types = {'BASE TABLE': 't', 'VIEW': 'v'} - return [TableInfo(row[0], types.get(row[1])) - for row in cursor.fetchall() - if row[0] not in self.ignored_tables] + if VERSION >= (4, 2): + return [TableInfo(row[0], types.get(row[1]), row[2]) + for row in cursor.fetchall() + if row[0] not in self.ignored_tables] + else: + return [BaseTableInfo(row[0], types.get(row[1])) + for row in cursor.fetchall() + if row[0] not in self.ignored_tables] def _is_auto_field(self, cursor, table_name, column_name): """ @@ -111,7 +129,7 @@ def get_table_description(self, cursor, table_name, identity_check=True): if not columns: raise DatabaseError(f"Table {table_name} does not exist.") - + items = [] for column in columns: if VERSION >= (3, 2): From e9383603fffa7915aafee4cf2e4511100cdc03cd Mon Sep 17 00:00:00 2001 From: mShan0 Date: Tue, 23 May 2023 15:57:20 -0700 Subject: [PATCH 18/31] add column comment to `get_table_description()` --- mssql/introspection.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 5e0feadd..1e022f51 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -8,7 +8,7 @@ from django import VERSION from django.db.backends.base.introspection import BaseDatabaseIntrospection -from django.db.backends.base.introspection import FieldInfo +from django.db.backends.base.introspection import FieldInfo as BaseFieldInfo from django.db.backends.base.introspection import TableInfo as BaseTableInfo from django.db.models.indexes import Index from django.conf import settings @@ -17,6 +17,7 @@ SQL_BIGAUTOFIELD = -777444 SQL_TIMESTAMP_WITH_TIMEZONE = -155 +FieldInfo = namedtuple("FieldInfo", BaseFieldInfo._fields + ("comment",)) TableInfo = namedtuple("TableInfo", BaseTableInfo._fields + ("comment",)) def get_schema_name(): @@ -144,7 +145,16 @@ def get_table_description(self, cursor, table_name, identity_check=True): column.append(collation_name[0] if collation_name else '') else: column.append('') - + if VERSION >= (4, 2): + sql = """select CAST(ep.value AS VARCHAR) AS COMMENT + FROM sys.columns c + INNER JOIN sys.tables t ON c.object_id = t.object_id + INNER JOIN sys.extended_properties ep ON c.object_id=ep.major_id AND ep.minor_id = c.column_id + WHERE t.name = '%s' AND c.name = '%s' AND ep.name = 'MS_Description' + """ % (table_name, column[0]) + cursor.execute(sql) + comment = cursor.fetchone() + column.append(comment) if identity_check and self._is_auto_field(cursor, table_name, column[0]): if column[1] == Database.SQL_BIGINT: column[1] = SQL_BIGAUTOFIELD @@ -152,7 +162,10 @@ def get_table_description(self, cursor, table_name, identity_check=True): column[1] = SQL_AUTOFIELD if column[1] == Database.SQL_WVARCHAR and column[3] < 4000: column[1] = Database.SQL_WCHAR - items.append(FieldInfo(*column)) + if VERSION >= (4, 2): + items.append(FieldInfo(*column)) + else: + items.append(BaseFieldInfo(*column)) return items def get_sequences(self, cursor, table_name, table_fields=()): From 852d67daf340effa551242f1d3dac9192ab2da9d Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 25 May 2023 12:37:22 -0700 Subject: [PATCH 19/31] return column comment only for `get_table_description()` --- mssql/introspection.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 1e022f51..16381fdf 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -154,7 +154,7 @@ def get_table_description(self, cursor, table_name, identity_check=True): """ % (table_name, column[0]) cursor.execute(sql) comment = cursor.fetchone() - column.append(comment) + column.append(comment[0] if comment else '') if identity_check and self._is_auto_field(cursor, table_name, column[0]): if column[1] == Database.SQL_BIGINT: column[1] = SQL_BIGAUTOFIELD From 0a2664ad020a426f0ab18a79526194c778e7ab95 Mon Sep 17 00:00:00 2001 From: Khanh Bui <85855766+khanhmaibui@users.noreply.github.com> Date: Tue, 30 May 2023 15:03:39 -0700 Subject: [PATCH 20/31] Add skipped tests to Django 4.2 (#268) * skip django 4.2 failing tests * skip schema test * skip aggregate annotation pruning test --------- Co-authored-by: mShan0 --- testapp/settings.py | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/testapp/settings.py b/testapp/settings.py index 9593692d..a2429a8b 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -291,7 +291,17 @@ 'model_fields.test_jsonfield.TestQuerying.test_has_key_number', # Django 4.2 - 'get_or_create.tests.UpdateOrCreateTests.test_update_only_defaults_and_pre_save_fields_when_local_fields' + 'get_or_create.tests.UpdateOrCreateTests.test_update_only_defaults_and_pre_save_fields_when_local_fields', + 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_empty_condition', + 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_multiple_subquery_annotation', + 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_subquery_annotation', + "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_group_by_annotation_kept" + 'aggregation.tests.AggregateTestCase.test_group_by_nested_expression_with_params', + 'expressions.tests.BasicExpressionsTests.test_aggregate_subquery_annotation', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_union_order_with_null_first_last', + 'queries.test_qs_combinators.QuerySetSetOperationTests.test_union_with_select_related_and_order', + 'expressions_window.tests.WindowFunctionTests.test_limited_filter', + 'schema.tests.SchemaTests.test_remove_ignored_unique_constraint_not_create_fk_index', ] REGEX_TESTS = [ From a3e895bc8691e9d2557e202258a0fca2f84a7072 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Wed, 31 May 2023 09:55:40 -0700 Subject: [PATCH 21/31] syntax fix --- testapp/settings.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/testapp/settings.py b/testapp/settings.py index 61a18613..ecdafbde 100644 --- a/testapp/settings.py +++ b/testapp/settings.py @@ -292,7 +292,7 @@ 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_empty_condition', 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_multiple_subquery_annotation', 'aggregation.test_filter_argument.FilteredAggregateTests.test_filtered_aggregate_ref_subquery_annotation', - "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_group_by_annotation_kept" + "aggregation.tests.AggregateAnnotationPruningTests.test_referenced_group_by_annotation_kept", 'aggregation.tests.AggregateTestCase.test_group_by_nested_expression_with_params', 'expressions.tests.BasicExpressionsTests.test_aggregate_subquery_annotation', 'queries.test_qs_combinators.QuerySetSetOperationTests.test_union_order_with_null_first_last', From 19e15390a4922222ad311c68c16aed36a23ff304 Mon Sep 17 00:00:00 2001 From: mShan0 Date: Wed, 31 May 2023 12:31:09 -0700 Subject: [PATCH 22/31] ci fix --- azure-pipelines.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/azure-pipelines.yml b/azure-pipelines.yml index ffd360a6..7cb512dd 100644 --- a/azure-pipelines.yml +++ b/azure-pipelines.yml @@ -133,10 +133,10 @@ jobs: matrix: Python3.11 - Django 4.2: python.version: '3.11' - tox.env: 'py310-django42' + tox.env: 'py311-django42' Python3.10 - Django 4.2: python.version: '3.10' - tox.env: 'py310-django42' + tox.env: 'py310-django42' Python 3.9 - Django 4.2: python.version: '3.9' tox.env: 'py39-django42' From 2885672004ff875fe32e60df2bb8daa5eafd0ffe Mon Sep 17 00:00:00 2001 From: mShan0 <96149598+mShan0@users.noreply.github.com> Date: Wed, 31 May 2023 14:03:45 -0700 Subject: [PATCH 23/31] bump version to 1.3 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 82d5ee0e..78532e94 100644 --- a/setup.py +++ b/setup.py @@ -27,7 +27,7 @@ setup( name='mssql-django', - version='1.2', + version='1.3', description='Django backend for Microsoft SQL Server', long_description=long_description, long_description_content_type='text/markdown', From 9c3be2b189e32c77696d64adf6729e94e5aadc6d Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Mon, 24 Jul 2023 20:57:13 -0700 Subject: [PATCH 24/31] add partial support for adding/altering comments on columns and tables --- mssql/schema.py | 68 +++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 66 insertions(+), 2 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 00ee5403..b171048d 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,6 +93,31 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" + sql_alter_table_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.TABLES i + INNER JOIN sys.tables t ON t.name = i.TABLE_NAME + LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id + WHERE (ep.name = 'MS_Description' AND ep.minor_id = 0)) + EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s + ELSE + EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s;""" + + sql_alter_column_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.COLUMNS i + INNER JOIN sys.columns t ON t.name = i.COLUMN_NAME + LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id + WHERE (ep.name = N'MS_Description' AND ep.minor_id = column_id)) + EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s + ELSE + EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, + @level0type = N'SCHEMA', @level0name = N'dbo', + @level1type = N'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s; """ _deferred_unique_indexes = defaultdict(list) @@ -316,7 +341,11 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Drop any FK constraints, we'll remake them later fks_dropped = set() - if old_field.remote_field and old_field.db_constraint: + if old_field.remote_field and old_field.db_constraint and self._field_should_be_altered( + old_field, + new_field, + ignore={"db_comment"}, + ): # Drop index, SQL Server requires explicit deletion if not hasattr(new_field, 'db_constraint') or not new_field.db_constraint: index_names = self._constraint_names(model, [old_field.column], index=True) @@ -447,7 +476,10 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, null_actions = [] post_actions = [] # Type change? - if old_type != new_type: + if old_type != new_type or ( + self.connection.features.supports_comments + and old_field.db_comment != new_field.db_comment + ): if django_version >= (4, 2): fragment, other_actions = self._alter_column_type_sql( model, old_field, new_field, new_type, old_collation=None, new_collation=None @@ -910,6 +942,18 @@ def add_field(self, model, field): "changes": changes_sql, } self.execute(sql, params) + # Add field comment, if required. + # if ( + # field.db_comment + # and self.connection.features.supports_comments + # and not self.connection.features.supports_comments_inline + # ): + # field_type = db_params["type"] + # self.execute( + # *self._alter_column_comment_sql( + # model, field, field_type, field.db_comment + # ) + # ) # Add an index, if required self.deferred_sql.extend(self._field_indexes_sql(model, field)) # Add any FK constraints later @@ -1117,6 +1161,23 @@ def create_model(self, model): # Prevent using [] as params, in the case a literal '%' is used in the definition self.execute(sql, params or None) + if self.connection.features.supports_comments: + # Add table comment. + if model._meta.db_table_comment: + self.alter_db_table_comment(model, None, model._meta.db_table_comment) + # Add column comments. + # if not self.connection.features.supports_comments_inline: + # for field in model._meta.local_fields: + # if field.db_comment: + # field_db_params = field.db_parameters( + # connection=self.connection + # ) + # field_type = field_db_params["type"] + # self.execute( + # *self._alter_column_comment_sql( + # model, field, field_type, field.db_comment + # ) + # ) # Add any field index and index_together's (deferred as SQLite3 _remake_table needs it) self.deferred_sql.extend(self._model_indexes_sql(model)) self.deferred_sql = list(set(self.deferred_sql)) @@ -1291,3 +1352,6 @@ def _create_index_name(self, table_name, column_names, suffix=""): new_index_name = index_name.replace('[', '').replace(']', '').replace('.', '_') return new_index_name return index_name + + def _alter_column_comment_sql(self, model, new_field, new_type, new_db_comment): + return "", [] \ No newline at end of file From be3616124d49e8f342c211caccc5ef4410414afc Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 27 Jul 2023 12:51:33 -0700 Subject: [PATCH 25/31] fix sql --- mssql/schema.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index b171048d..6483bdc3 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -108,16 +108,16 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_alter_column_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.COLUMNS i INNER JOIN sys.columns t ON t.name = i.COLUMN_NAME LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE (ep.name = N'MS_Description' AND ep.minor_id = column_id)) + WHERE (ep.name = N'MS_Description' AND ep.minor_id = 2)) EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s, - @level2type = 'COLUMN', @level2name = %(column)s + @level2type = N'COLUMN', @level2name = %(column)s ELSE EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s, - @level2type = 'COLUMN', @level2name = %(column)s; """ + @level2type = N'COLUMN', @level2name = %(column)s; """ _deferred_unique_indexes = defaultdict(list) From 8f9f00a09f13e8f8cda3c90a1cf23c426bd5f92e Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 27 Jul 2023 13:02:36 -0700 Subject: [PATCH 26/31] add django 4.2 condition and fix sql --- mssql/schema.py | 23 +++++++---------------- 1 file changed, 7 insertions(+), 16 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 6483bdc3..6899c2df 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -105,19 +105,10 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s;""" - sql_alter_column_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.COLUMNS i - INNER JOIN sys.columns t ON t.name = i.COLUMN_NAME - LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE (ep.name = N'MS_Description' AND ep.minor_id = 2)) - EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + sql_alter_column_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', @level1type = N'TABLE', @level1name = %(table)s, - @level2type = N'COLUMN', @level2name = %(column)s - ELSE - EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s, - @level2type = N'COLUMN', @level2name = %(column)s; """ + @level2type = N'COLUMN', @level2name = %(column)s""" _deferred_unique_indexes = defaultdict(list) @@ -341,11 +332,11 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Drop any FK constraints, we'll remake them later fks_dropped = set() - if old_field.remote_field and old_field.db_constraint and self._field_should_be_altered( + if old_field.remote_field and old_field.db_constraint and (django_version >= (4, 2) and self._field_should_be_altered( old_field, new_field, ignore={"db_comment"}, - ): + )): # Drop index, SQL Server requires explicit deletion if not hasattr(new_field, 'db_constraint') or not new_field.db_constraint: index_names = self._constraint_names(model, [old_field.column], index=True) @@ -475,8 +466,8 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, actions = [] null_actions = [] post_actions = [] - # Type change? - if old_type != new_type or ( + # Type or comment change? + if old_type != new_type or (django_version >= (4, 2) and self.connection.features.supports_comments and old_field.db_comment != new_field.db_comment ): @@ -1161,7 +1152,7 @@ def create_model(self, model): # Prevent using [] as params, in the case a literal '%' is used in the definition self.execute(sql, params or None) - if self.connection.features.supports_comments: + if django_version >= (4, 2) and self.connection.features.supports_comments: # Add table comment. if model._meta.db_table_comment: self.alter_db_table_comment(model, None, model._meta.db_table_comment) From f9190bec47b8d3d2d8cb2d52a669722fdf706033 Mon Sep 17 00:00:00 2001 From: Khanh Bui Date: Thu, 27 Jul 2023 13:46:35 -0700 Subject: [PATCH 27/31] fix sql --- mssql/schema.py | 12 ++---------- 1 file changed, 2 insertions(+), 10 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 6899c2df..7f0bf153 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,17 +93,9 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" - sql_alter_table_comment = """IF NOT EXISTS (SELECT NULL FROM INFORMATION_SCHEMA.TABLES i - INNER JOIN sys.tables t ON t.name = i.TABLE_NAME - LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE (ep.name = 'MS_Description' AND ep.minor_id = 0)) - EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, + sql_alter_table_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s - ELSE - EXECUTE sp_updateextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s;""" + @level1type = N'TABLE', @level1name = %(table)s""" sql_alter_column_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, @level0type = N'SCHEMA', @level0name = N'dbo', From 25900238b2c5b856dec6973dbb9e00e40aee6c4d Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Thu, 7 Dec 2023 17:49:49 -0800 Subject: [PATCH 28/31] Implement db_comment --- mssql/introspection.py | 2 +- mssql/schema.py | 94 ++++++++++++++++++++++++++---------------- 2 files changed, 60 insertions(+), 36 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 16381fdf..86f217e5 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -83,7 +83,7 @@ def get_table_list(self, cursor): INNER JOIN sys.tables t ON t.name = i.TABLE_NAME LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id WHERE - ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id = 0) OR ep.value IS NULL) + ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id >= 0) OR ep.value IS NULL) AND i.TABLE_SCHEMA = %s""" % ( get_schema_name()) diff --git a/mssql/schema.py b/mssql/schema.py index 7f0bf153..35ea60e8 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -93,15 +93,40 @@ class DatabaseSchemaEditor(BaseDatabaseSchemaEditor): sql_rename_table = "EXEC sp_rename %(old_table)s, %(new_table)s" sql_create_unique_null = "CREATE UNIQUE INDEX %(name)s ON %(table)s(%(columns)s) " \ "WHERE %(columns)s IS NOT NULL" - sql_alter_table_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s""" - - sql_alter_column_comment = """EXECUTE sp_addextendedproperty @name = N'MS_Description', @value = %(comment)s, - @level0type = N'SCHEMA', @level0name = N'dbo', - @level1type = N'TABLE', @level1name = %(table)s, - @level2type = N'COLUMN', @level2name = %(column)s""" - + sql_alter_table_comment= """ + IF NOT EXISTS (SELECT NULL FROM sys.extended_properties ep + WHERE ep.major_id = OBJECT_ID('%(table)s') + AND ep.name = 'MS_Description' + AND ep.minor_id = 0) + EXECUTE sp_addextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s + ELSE + EXECUTE sp_updateextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s + """ + sql_alter_column_comment= """ + IF NOT EXISTS (SELECT NULL FROM sys.extended_properties ep + WHERE ep.major_id = OBJECT_ID('%(table)s') + AND ep.name = 'MS_Description' + AND ep.minor_id = (SELECT column_id FROM sys.columns + WHERE name = '%(column)s' + AND object_id = OBJECT_ID('%(table)s'))) + EXECUTE sp_addextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s + ELSE + EXECUTE sp_updateextendedproperty + @name = 'MS_Description', @value = %(comment)s, + @level0type = 'SCHEMA', @level0name = 'dbo', + @level1type = 'TABLE', @level1name = %(table)s, + @level2type = 'COLUMN', @level2name = %(column)s + """ _deferred_unique_indexes = defaultdict(list) def _alter_column_default_sql(self, model, old_field, new_field, drop=False): @@ -172,6 +197,8 @@ def _alter_column_null_sql(self, model, old_field, new_field): if django_version >= (4, 2): def _alter_column_type_sql(self, model, old_field, new_field, new_type, old_collation, new_collation): new_type = self._set_field_new_type_null_status(old_field, new_type) + # Check if existing + # Drop exisiting return super()._alter_column_type_sql(model, old_field, new_field, new_type, old_collation, new_collation) else: def _alter_column_type_sql(self, model, old_field, new_field, new_type): @@ -926,17 +953,17 @@ def add_field(self, model, field): } self.execute(sql, params) # Add field comment, if required. - # if ( - # field.db_comment - # and self.connection.features.supports_comments - # and not self.connection.features.supports_comments_inline - # ): - # field_type = db_params["type"] - # self.execute( - # *self._alter_column_comment_sql( - # model, field, field_type, field.db_comment - # ) - # ) + if ( + field.db_comment + and self.connection.features.supports_comments + and not self.connection.features.supports_comments_inline + ): + field_type = db_params["type"] + self.execute( + *self._alter_column_comment_sql( + model, field, field_type, field.db_comment + ) + ) # Add an index, if required self.deferred_sql.extend(self._field_indexes_sql(model, field)) # Add any FK constraints later @@ -1149,18 +1176,18 @@ def create_model(self, model): if model._meta.db_table_comment: self.alter_db_table_comment(model, None, model._meta.db_table_comment) # Add column comments. - # if not self.connection.features.supports_comments_inline: - # for field in model._meta.local_fields: - # if field.db_comment: - # field_db_params = field.db_parameters( - # connection=self.connection - # ) - # field_type = field_db_params["type"] - # self.execute( - # *self._alter_column_comment_sql( - # model, field, field_type, field.db_comment - # ) - # ) + if not self.connection.features.supports_comments_inline: + for field in model._meta.local_fields: + if field.db_comment: + field_db_params = field.db_parameters( + connection=self.connection + ) + field_type = field_db_params["type"] + self.execute( + *self._alter_column_comment_sql( + model, field, field_type, field.db_comment + ) + ) # Add any field index and index_together's (deferred as SQLite3 _remake_table needs it) self.deferred_sql.extend(self._model_indexes_sql(model)) self.deferred_sql = list(set(self.deferred_sql)) @@ -1335,6 +1362,3 @@ def _create_index_name(self, table_name, column_names, suffix=""): new_index_name = index_name.replace('[', '').replace(']', '').replace('.', '_') return new_index_name return index_name - - def _alter_column_comment_sql(self, model, new_field, new_type, new_db_comment): - return "", [] \ No newline at end of file From 965573a80695a12da35417fec5535080aa8eb928 Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Fri, 8 Dec 2023 10:52:14 -0800 Subject: [PATCH 29/31] Update get_table_list and add version condition to db_comment --- mssql/introspection.py | 8 +++----- mssql/schema.py | 23 +++++++++++------------ 2 files changed, 14 insertions(+), 17 deletions(-) diff --git a/mssql/introspection.py b/mssql/introspection.py index 713dd07e..efba52ad 100644 --- a/mssql/introspection.py +++ b/mssql/introspection.py @@ -82,12 +82,10 @@ def get_table_list(self, cursor): TABLE_TYPE, CAST(ep.value AS VARCHAR) AS COMMENT FROM INFORMATION_SCHEMA.TABLES i - INNER JOIN sys.tables t ON t.name = i.TABLE_NAME + LEFT JOIN sys.tables t ON t.name = i.TABLE_NAME LEFT JOIN sys.extended_properties ep ON t.object_id = ep.major_id - WHERE - ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id >= 0) OR ep.value IS NULL) - AND - i.TABLE_SCHEMA = %s""" % ( + AND ((ep.name = 'MS_DESCRIPTION' AND ep.minor_id = 0) OR ep.value IS NULL) + AND i.TABLE_SCHEMA = %s""" % ( get_schema_name()) cursor.execute(sql) types = {'BASE TABLE': 't', 'VIEW': 'v'} diff --git a/mssql/schema.py b/mssql/schema.py index 34d347a7..1768b4f9 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -197,8 +197,6 @@ def _alter_column_null_sql(self, model, old_field, new_field): if django_version >= (4, 2): def _alter_column_type_sql(self, model, old_field, new_field, new_type, old_collation, new_collation): new_type = self._set_field_new_type_null_status(old_field, new_type) - # Check if existing - # Drop exisiting return super()._alter_column_type_sql(model, old_field, new_field, new_type, old_collation, new_collation) else: def _alter_column_type_sql(self, model, old_field, new_field, new_type): @@ -965,17 +963,18 @@ def add_field(self, model, field): } self.execute(sql, params) # Add field comment, if required. - if ( - field.db_comment - and self.connection.features.supports_comments - and not self.connection.features.supports_comments_inline - ): - field_type = db_params["type"] - self.execute( - *self._alter_column_comment_sql( - model, field, field_type, field.db_comment + if django_version >= (4, 2): + if ( + field.db_comment + and self.connection.features.supports_comments + and not self.connection.features.supports_comments_inline + ): + field_type = db_params["type"] + self.execute( + *self._alter_column_comment_sql( + model, field, field_type, field.db_comment + ) ) - ) # Add an index, if required self.deferred_sql.extend(self._field_indexes_sql(model, field)) # Add any FK constraints later From 37d48d0bee0c8005b904f0e5e3347bec5e83cafb Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Mon, 11 Dec 2023 10:47:32 -0800 Subject: [PATCH 30/31] Fix drop fk condition --- mssql/schema.py | 18 +++++++++++++----- 1 file changed, 13 insertions(+), 5 deletions(-) diff --git a/mssql/schema.py b/mssql/schema.py index 1768b4f9..42b64ec5 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -349,11 +349,19 @@ def _alter_field(self, model, old_field, new_field, old_type, new_type, # Drop any FK constraints, we'll remake them later fks_dropped = set() - if old_field.remote_field and old_field.db_constraint and (django_version >= (4, 2) and self._field_should_be_altered( - old_field, - new_field, - ignore={"db_comment"}, - )): + if ( + old_field.remote_field + and old_field.db_constraint + and (django_version < (4,2) + or + (django_version >= (4, 2) + and self._field_should_be_altered( + old_field, + new_field, + ignore={"db_comment"}) + ) + ) + ): # Drop index, SQL Server requires explicit deletion if not hasattr(new_field, 'db_constraint') or not new_field.db_constraint: index_names = self._constraint_names(model, [old_field.column], index=True) From bb2cb087e2b506d6a5671deff820fc55f2fc8609 Mon Sep 17 00:00:00 2001 From: "Daniel Au (SIMBA TECHNOLOGIES INC)" Date: Mon, 11 Dec 2023 14:42:24 -0800 Subject: [PATCH 31/31] Fix alter comment behavior --- mssql/schema.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/mssql/schema.py b/mssql/schema.py index 42b64ec5..771b93b6 100644 --- a/mssql/schema.py +++ b/mssql/schema.py @@ -171,7 +171,18 @@ def _alter_column_default_sql(self, model, old_field, new_field, drop=False): }, params, ) - + + def _alter_column_comment_sql(self, model, new_field, new_type, new_db_comment): + return ( + self.sql_alter_column_comment + % { + "table": self.quote_name(model._meta.db_table), + "column": new_field.column, + "comment": self._comment_sql(new_db_comment), + }, + [], + ) + def _alter_column_null_sql(self, model, old_field, new_field): """ Hook to specialize column null alteration.