Skip to content

Commit 9fdac8a

Browse files
authored
Ensure source node .freshness is equal to node's .config.freshness (#11719) (#11721)
* Ensure source node `.freshness` is equal to node's `.config.freshness` * Default source config freshness to empty spec if no freshenss spec is given * Update contract tests for source nodes
1 parent c20c71a commit 9fdac8a

File tree

6 files changed

+60
-5
lines changed

6 files changed

+60
-5
lines changed
Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
kind: Fixes
2+
body: Ensure source node `.freshness` is equal to node's `.config.freshness`
3+
time: 2025-06-09T17:52:39.978403-05:00
4+
custom:
5+
Author: QMalcolm
6+
Issue: "11717"

core/dbt/artifacts/resources/v1/source_definition.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -20,7 +20,7 @@
2020
class SourceConfig(BaseConfig):
2121
enabled: bool = True
2222
event_time: Any = None
23-
freshness: Optional[FreshnessThreshold] = None
23+
freshness: Optional[FreshnessThreshold] = field(default_factory=FreshnessThreshold)
2424

2525

2626
@dataclass

core/dbt/parser/sources.py

Lines changed: 1 addition & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -184,8 +184,7 @@ def parse_source(self, target: UnpatchedSourceDefinition) -> SourceDefinition:
184184
meta=meta,
185185
loader=source.loader,
186186
loaded_at_field=loaded_at_field,
187-
# The setting to an empty freshness object is to maintain what we were previously doing if no freshenss was specified
188-
freshness=config.freshness or FreshnessThreshold(),
187+
freshness=config.freshness,
189188
quoting=quoting,
190189
resource_type=NodeType.Source,
191190
fqn=target.fqn,

tests/functional/artifacts/expected_manifest.py

Lines changed: 10 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -792,7 +792,11 @@ def expected_seeded_manifest(project, model_database=None, quote_model=False):
792792
"config": {
793793
"enabled": True,
794794
"event_time": None,
795-
"freshness": None,
795+
"freshness": {
796+
"error_after": {"count": None, "period": None},
797+
"warn_after": {"count": None, "period": None},
798+
"filter": None,
799+
},
796800
},
797801
"quoting": {
798802
"database": None,
@@ -1308,7 +1312,11 @@ def expected_references_manifest(project):
13081312
"config": {
13091313
"enabled": True,
13101314
"event_time": None,
1311-
"freshness": None,
1315+
"freshness": {
1316+
"error_after": {"count": None, "period": None},
1317+
"warn_after": {"count": None, "period": None},
1318+
"filter": None,
1319+
},
13121320
},
13131321
"quoting": {
13141322
"database": False,

tests/unit/contracts/graph/test_nodes_parsed.py

Lines changed: 8 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1886,6 +1886,10 @@ def basic_parsed_source_definition_dict():
18861886
"tags": [],
18871887
"config": {
18881888
"enabled": True,
1889+
"freshness": {
1890+
"warn_after": {},
1891+
"error_after": {},
1892+
},
18891893
},
18901894
"unrendered_config": {},
18911895
}
@@ -1916,6 +1920,10 @@ def complex_parsed_source_definition_dict():
19161920
"tags": ["my_tag"],
19171921
"config": {
19181922
"enabled": True,
1923+
"freshness": {
1924+
"warn_after": {},
1925+
"error_after": {},
1926+
},
19191927
},
19201928
"freshness": {"warn_after": {"period": "hour", "count": 1}, "error_after": {}},
19211929
"loaded_at_field": "loaded_at",

tests/unit/parser/test_parser.py

Lines changed: 34 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -404,10 +404,31 @@ def assertEqualNodes(node_one, node_two):
404404
- unique
405405
"""
406406

407+
SOURCE_FRESHNESS_AT_TABLE_AND_CONFIG = """
408+
sources:
409+
- name: my_source
410+
loaded_at_field: test
411+
tables:
412+
- name: my_table
413+
freshness:
414+
warn_after: {count: 1, period: hour}
415+
error_after: {count: 1, period: day}
416+
config:
417+
freshness:
418+
warn_after: {count: 2, period: hour}
419+
error_after: {count: 2, period: day}
420+
"""
421+
407422

408423
class SchemaParserTest(BaseParserTest):
409424
def setUp(self):
410425
super().setUp()
426+
# Reset `warn_error` to False so we don't raise warnigns about top level freshness as errors
427+
set_from_args(
428+
Namespace(warn_error=False, state_modified_compare_more_unrendered_values=False),
429+
None,
430+
)
431+
411432
self.parser = SchemaParser(
412433
project=self.snowplow_project_config,
413434
manifest=self.manifest,
@@ -448,6 +469,19 @@ def test__read_basic_source(self):
448469
self.assertEqual(source_values[0].table.description, "")
449470
self.assertEqual(len(source_values[0].table.columns), 0)
450471

472+
@mock.patch("dbt.parser.sources.get_adapter")
473+
def test_parse_source_resulting_node_freshness_matches_config_freshness(self, _):
474+
block = self.file_block_for(SOURCE_FRESHNESS_AT_TABLE_AND_CONFIG, "test_one.yml")
475+
dct = yaml_from_file(block.file)
476+
self.parser.parse_file(block, dct)
477+
unpatched_src_default = self.parser.manifest.sources["source.snowplow.my_source.my_table"]
478+
src_default = self.source_patcher.parse_source(unpatched_src_default)
479+
assert src_default.freshness == src_default.config.freshness
480+
assert src_default.freshness.warn_after.count == 2
481+
assert src_default.freshness.warn_after.period == "hour"
482+
assert src_default.freshness.error_after.count == 2
483+
assert src_default.freshness.error_after.period == "day"
484+
451485
def test__parse_basic_source(self):
452486
block = self.file_block_for(SINGLE_TABLE_SOURCE, "test_one.yml")
453487
dct = yaml_from_file(block.file)

0 commit comments

Comments
 (0)