Skip to content

Commit

Permalink
Fix fast partition count calculations for daily partitions definition (
Browse files Browse the repository at this point in the history
…#22046)

Summary:
owen tried to warn me about this on
https://github.com/dagster-io/dagster/pull/21791/files, and in my hubris
I did not listen - the test coverage here was not as thorough as i
thought it was. Fix the fast calculations when you have a simple daily
partitions definition (the existing logic gets messed up by DST
boundaries when they are on opposite sides of the boundary).

Test Plan: BK, shadow graphql of an incorrect partitions count

## Summary & Motivation

## How I Tested These Changes
  • Loading branch information
gibsondan authored May 23, 2024
1 parent 9bbc980 commit a543670
Show file tree
Hide file tree
Showing 2 changed files with 33 additions and 2 deletions.
Original file line number Diff line number Diff line change
Expand Up @@ -3,7 +3,7 @@
import json
import re
from abc import abstractmethod, abstractproperty
from datetime import datetime, timedelta
from datetime import date, datetime, timedelta
from enum import Enum
from functools import cached_property
from typing import (
Expand Down Expand Up @@ -383,7 +383,18 @@ def _get_fast_num_partitions(self, current_time: Optional[datetime] = None) -> O
return None

if self.is_basic_daily:
return (last_partition_window.start - first_partition_window.start).days + 1
return (
date(
last_partition_window.start.year,
last_partition_window.start.month,
last_partition_window.start.day,
)
- date(
first_partition_window.start.year,
first_partition_window.start.month,
first_partition_window.start.day,
)
).days + 1

fixed_minute_interval = get_fixed_minute_interval(self.cron_schedule)
if fixed_minute_interval:
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -1240,6 +1240,26 @@ def my_partitioned_config_2(_start, _end):
partitions_def.get_partition_keys(current_time)
)

@daily_partitioned_config(start_date="2020-01-01", timezone="US/Pacific")
def my_daily_dst_transition_partitioned_config(_start, _end):
return {}

partitions_def = cast(
TimeWindowPartitionsDefinition, my_daily_dst_transition_partitioned_config.partitions_def
)

current_time_post_transition = datetime.strptime("2024-05-22", "%Y-%m-%d")

assert partitions_def.get_num_partitions(current_time_post_transition) == len(
partitions_def.get_partition_keys(current_time_post_transition)
)

current_time_pre_transition = datetime.strptime("2024-02-01", "%Y-%m-%d")

assert partitions_def.get_num_partitions(current_time_pre_transition) == len(
partitions_def.get_partition_keys(current_time_pre_transition)
)


def test_get_first_partition_window():
assert DailyPartitionsDefinition(
Expand Down

0 comments on commit a543670

Please sign in to comment.