diff --git a/src/backend/base/langflow/schema/encoders.py b/src/backend/base/langflow/schema/encoders.py index 93b6af740d..d45d34abb8 100644 --- a/src/backend/base/langflow/schema/encoders.py +++ b/src/backend/base/langflow/schema/encoders.py @@ -7,7 +7,7 @@ def encode_callable(obj: Callable): def encode_datetime(obj: datetime): - return obj.strftime("%Y-%m-%d %H:%M:%S %Z") + return obj.strftime("%Y-%m-%d %H:%M:%S.%f %Z") CUSTOM_ENCODERS = {Callable: encode_callable, datetime: encode_datetime} diff --git a/src/backend/base/langflow/schema/playground_events.py b/src/backend/base/langflow/schema/playground_events.py index 426e5759ca..c80b957ebd 100644 --- a/src/backend/base/langflow/schema/playground_events.py +++ b/src/backend/base/langflow/schema/playground_events.py @@ -21,7 +21,7 @@ class PlaygroundEvent(BaseModel): files: list[str] | None = Field(default=None) text: str | None = Field(default=None) timestamp: Annotated[str, timestamp_to_str_validator] = Field( - default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f %Z") ) id_: UUID | str | None = Field(default=None, alias="id") @@ -80,7 +80,7 @@ class TokenEvent(BaseModel): chunk: str = Field(...) id: UUID | str | None = Field(alias="id") timestamp: Annotated[str, timestamp_to_str_validator] = Field( - default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f %Z") ) diff --git a/src/backend/base/langflow/schema/validators.py b/src/backend/base/langflow/schema/validators.py index b53ce86df5..befafe808d 100644 --- a/src/backend/base/langflow/schema/validators.py +++ b/src/backend/base/langflow/schema/validators.py @@ -2,6 +2,24 @@ from datetime import datetime, timezone from pydantic import BeforeValidator +TF_WITH_TZ_AND_MICROSECONDS = "%Y-%m-%d %H:%M:%S.%f %Z" +TF_WITH_TZ_AND_MICROSECONDS_ISO = "%Y-%m-%dT%H:%M:%S.%f%z" + +# An ordered list of timestamp formats to try to parse from str, from most to least specific +TIMESTAMP_FORMATS = [ + TF_WITH_TZ_AND_MICROSECONDS, # Standard with timezone and microseconds + TF_WITH_TZ_AND_MICROSECONDS_ISO, # ISO with numeric timezone and microseconds + "%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds + "%Y-%m-%d %H:%M:%S.%f", # Without timezone, with microseconds + "%Y-%m-%d %H:%M:%S %Z", # Standard with timezone + "%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone + "%Y-%m-%dT%H:%M:%S", # ISO format + "%Y-%m-%d %H:%M:%S", # Without timezone +] + +# Formats that carry their own timezone offset — must use astimezone, not replace +_FORMATS_WITH_NUMERIC_TZ = {TF_WITH_TZ_AND_MICROSECONDS_ISO, "%Y-%m-%dT%H:%M:%S%z"} + def timestamp_to_str(timestamp: datetime | str) -> str: """Convert timestamp to standardized string format. @@ -12,25 +30,20 @@ def timestamp_to_str(timestamp: datetime | str) -> str: timestamp (datetime | str): Input timestamp either as datetime object or string Returns: - str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS UTC' format + str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS.ffffff UTC' format Raises: ValueError: If string timestamp is in invalid format """ if isinstance(timestamp, str): - # Try parsing with different formats - formats = [ - "%Y-%m-%dT%H:%M:%S", # ISO format - "%Y-%m-%d %H:%M:%S %Z", # Standard with timezone - "%Y-%m-%d %H:%M:%S", # Without timezone - "%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds - "%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone - ] - - for fmt in formats: + for fmt in TIMESTAMP_FORMATS: try: - parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc) - return parsed.strftime("%Y-%m-%d %H:%M:%S %Z") + parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007 + if fmt in _FORMATS_WITH_NUMERIC_TZ: + parsed = parsed.astimezone(timezone.utc) + else: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.strftime(TF_WITH_TZ_AND_MICROSECONDS) except ValueError: continue @@ -40,7 +53,7 @@ def timestamp_to_str(timestamp: datetime | str) -> str: # Handle datetime object if timestamp.tzinfo is None: timestamp = timestamp.replace(tzinfo=timezone.utc) - return timestamp.strftime("%Y-%m-%d %H:%M:%S %Z") + return timestamp.strftime(TF_WITH_TZ_AND_MICROSECONDS) def str_to_timestamp(timestamp: str | datetime) -> datetime: @@ -55,60 +68,22 @@ def str_to_timestamp(timestamp: str | datetime) -> datetime: datetime: Datetime object with UTC timezone Raises: - ValueError: If string timestamp is not in 'YYYY-MM-DD HH:MM:SS UTC' format + ValueError: If string timestamp is not in a recognised format """ if isinstance(timestamp, str): - try: - return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc) - except ValueError as e: - msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS UTC" - raise ValueError(msg) from e - return timestamp - - -def timestamp_with_fractional_seconds(timestamp: datetime | str) -> str: - """Convert timestamp to string format including fractional seconds. - - Handles multiple input formats and ensures consistent UTC timezone output. - - Args: - timestamp (datetime | str): Input timestamp either as datetime object or string - - Returns: - str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS.ffffff UTC' format - - Raises: - ValueError: If string timestamp is in invalid format - """ - if isinstance(timestamp, str): - # Try parsing with different formats - formats = [ - "%Y-%m-%d %H:%M:%S.%f %Z", # Standard with timezone - "%Y-%m-%d %H:%M:%S.%f", # Without timezone - "%Y-%m-%dT%H:%M:%S.%f", # ISO format - "%Y-%m-%dT%H:%M:%S.%f%z", # ISO with numeric timezone - # Also try without fractional seconds - "%Y-%m-%d %H:%M:%S %Z", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%dT%H:%M:%S", - ] - - for fmt in formats: + for fmt in TIMESTAMP_FORMATS: try: - parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc) - return parsed.strftime("%Y-%m-%d %H:%M:%S.%f %Z") + parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007 + if fmt in _FORMATS_WITH_NUMERIC_TZ: + return parsed.astimezone(timezone.utc) + return parsed.replace(tzinfo=timezone.utc) except ValueError: continue - msg = f"Invalid timestamp format: {timestamp}" + msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS.ffffff UTC" raise ValueError(msg) - - # Handle datetime object - if timestamp.tzinfo is None: - timestamp = timestamp.replace(tzinfo=timezone.utc) - return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f %Z") + return timestamp timestamp_to_str_validator = BeforeValidator(timestamp_to_str) -timestamp_with_fractional_seconds_validator = BeforeValidator(timestamp_with_fractional_seconds) str_to_timestamp_validator = BeforeValidator(str_to_timestamp) diff --git a/src/backend/base/langflow/services/database/models/message/model.py b/src/backend/base/langflow/services/database/models/message/model.py index a10291c27c..3839c14b89 100644 --- a/src/backend/base/langflow/services/database/models/message/model.py +++ b/src/backend/base/langflow/services/database/models/message/model.py @@ -10,7 +10,7 @@ from sqlmodel import JSON, Column, Field, SQLModel from langflow.schema.content_block import ContentBlock from langflow.schema.properties import Properties -from langflow.schema.validators import str_to_timestamp_validator +from langflow.schema.validators import TF_WITH_TZ_AND_MICROSECONDS, str_to_timestamp, str_to_timestamp_validator if TYPE_CHECKING: from langflow.schema.message import Message @@ -39,11 +39,11 @@ class MessageBase(SQLModel): if isinstance(value, datetime): if value.tzinfo is None: value = value.replace(tzinfo=timezone.utc) - return value.strftime("%Y-%m-%d %H:%M:%S %Z") + return value.strftime(TF_WITH_TZ_AND_MICROSECONDS) if isinstance(value, str): - value = datetime.fromisoformat(value).replace(tzinfo=timezone.utc) - return value.strftime("%Y-%m-%d %H:%M:%S %Z") + dt = str_to_timestamp(value) # unified, UTC-normalized + return dt.strftime(TF_WITH_TZ_AND_MICROSECONDS) return value @@ -85,10 +85,15 @@ class MessageBase(SQLModel): message.files = image_paths if isinstance(message.timestamp, str): + # Convert timestamp string in format "YYYY-MM-DD HH:MM:SS.ffffff UTC" to datetime try: - timestamp = datetime.strptime(message.timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc) + timestamp = datetime.strptime(message.timestamp, TF_WITH_TZ_AND_MICROSECONDS).replace( + tzinfo=timezone.utc + ) except ValueError: - timestamp = datetime.fromisoformat(message.timestamp).replace(tzinfo=timezone.utc) + # Fallback for ISO format if the above fails; astimezone preserves offset if present + parsed = datetime.fromisoformat(message.timestamp) + timestamp = parsed.astimezone(timezone.utc) if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc) else: timestamp = message.timestamp diff --git a/src/backend/tests/unit/schema/test_timestamp_validators.py b/src/backend/tests/unit/schema/test_timestamp_validators.py new file mode 100644 index 0000000000..194a2cb00d --- /dev/null +++ b/src/backend/tests/unit/schema/test_timestamp_validators.py @@ -0,0 +1,157 @@ +"""Unit tests for timestamp validator functions in both langflow and lfx schemas.""" + +from datetime import datetime, timezone + +import langflow.schema.validators as lf_validators +import lfx.schema.validators as lfx_validators +import pytest + +# --------------------------------------------------------------------------- +# Helpers +# --------------------------------------------------------------------------- + + +def _utc(year, month, day, hour, minute, second, microsecond=0): + return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=timezone.utc) + + +# --------------------------------------------------------------------------- +# Parametrize over both implementations so every test runs for both +# --------------------------------------------------------------------------- + +VALIDATOR_MODULES = [ + pytest.param(lf_validators, id="langflow"), + pytest.param(lfx_validators, id="lfx"), +] + + +@pytest.mark.parametrize("mod", VALIDATOR_MODULES) +class TestTimestampToStr: + """timestamp_to_str converts various inputs to the canonical string format.""" + + def test_datetime_with_microseconds_preserved(self, mod): + dt = _utc(2024, 1, 15, 10, 30, 45, 123456) + result = mod.timestamp_to_str(dt) + assert result == "2024-01-15 10:30:45.123456 UTC" + + def test_datetime_without_microseconds_zero_padded(self, mod): + dt = _utc(2024, 6, 1, 0, 0, 0, 0) + result = mod.timestamp_to_str(dt) + assert result == "2024-06-01 00:00:00.000000 UTC" + + def test_datetime_naive_treated_as_utc(self, mod): + dt = datetime(2024, 3, 10, 8, 0, 0, 500000) # noqa: DTZ001 — intentionally naive to test the fallback + result = mod.timestamp_to_str(dt) + assert result == "2024-03-10 08:00:00.500000 UTC" + + def test_string_old_format_no_microseconds(self, mod): + """Backward compat: old format without microseconds must still parse.""" + result = mod.timestamp_to_str("2024-06-15 10:30:00 UTC") + assert result == "2024-06-15 10:30:00.000000 UTC" + + def test_string_new_format_with_microseconds(self, mod): + result = mod.timestamp_to_str("2024-01-15 10:30:45.123456 UTC") + assert result == "2024-01-15 10:30:45.123456 UTC" + + def test_string_iso_with_numeric_tz_offset_positive(self, mod): + """Non-UTC numeric offset must be converted correctly, not silently corrupted.""" + # +05:00 means real UTC is 05:30:45 + result = mod.timestamp_to_str("2024-01-15T10:30:45+05:00") + assert result == "2024-01-15 05:30:45.000000 UTC" + + def test_string_iso_with_numeric_tz_offset_negative(self, mod): + # -03:00 means real UTC is 13:30:45 + result = mod.timestamp_to_str("2024-01-15T10:30:45-03:00") + assert result == "2024-01-15 13:30:45.000000 UTC" + + def test_string_invalid_format_raises(self, mod): + with pytest.raises(ValueError, match="Invalid timestamp format"): + mod.timestamp_to_str("not-a-date") + + +@pytest.mark.parametrize("mod", VALIDATOR_MODULES) +class TestStrToTimestamp: + """str_to_timestamp converts strings (and datetimes) back to datetime objects.""" + + def test_new_format_with_microseconds(self, mod): + result = mod.str_to_timestamp("2024-01-15 10:30:45.123456 UTC") + assert result == _utc(2024, 1, 15, 10, 30, 45, 123456) + + def test_old_format_without_microseconds_backward_compat(self, mod): + """Old-format strings must parse without crashing.""" + result = mod.str_to_timestamp("2024-06-15 10:30:00 UTC") + assert result == _utc(2024, 6, 15, 10, 30, 0, 0) + + def test_iso_with_numeric_tz_offset(self, mod): + result = mod.str_to_timestamp("2024-01-15T10:30:45+05:00") + assert result == _utc(2024, 1, 15, 5, 30, 45, 0) + + def test_passthrough_for_datetime_object(self, mod): + dt = _utc(2024, 5, 20, 12, 0, 0, 999) + assert mod.str_to_timestamp(dt) is dt + + def test_invalid_string_raises(self, mod): + with pytest.raises(ValueError, match="Invalid timestamp format"): + mod.str_to_timestamp("not-a-date") + + +@pytest.mark.parametrize("mod", VALIDATOR_MODULES) +class TestRoundtrip: + """timestamp_to_str ↔ str_to_timestamp must be lossless for microseconds.""" + + def test_microseconds_survive_roundtrip(self, mod): + original = _utc(2024, 11, 3, 22, 59, 59, 999999) + as_str = mod.timestamp_to_str(original) + recovered = mod.str_to_timestamp(as_str) + assert recovered == original + + def test_zero_microseconds_survive_roundtrip(self, mod): + original = _utc(2024, 1, 1, 0, 0, 0, 0) + as_str = mod.timestamp_to_str(original) + recovered = mod.str_to_timestamp(as_str) + assert recovered == original + + +@pytest.mark.parametrize("mod", VALIDATOR_MODULES) +class TestMessageOrdering: + """Two messages differing only by microseconds must maintain correct order.""" + + def test_sub_second_ordering(self, mod): + earlier = _utc(2024, 6, 15, 12, 0, 0, 1) + later = _utc(2024, 6, 15, 12, 0, 0, 999999) + + earlier_str = mod.timestamp_to_str(earlier) + later_str = mod.timestamp_to_str(later) + + # String comparison must preserve chronological order + assert earlier_str < later_str + + # And roundtrip back to datetime must also preserve order + assert mod.str_to_timestamp(earlier_str) < mod.str_to_timestamp(later_str) + + +class TestEncoderCompatibility: + """encode_datetime output must be parseable by both validator chains.""" + + def test_encode_datetime_parseable_by_langflow_str_to_timestamp(self): + from langflow.schema.encoders import encode_datetime + + dt = _utc(2024, 8, 20, 14, 0, 0, 654321) + encoded = encode_datetime(dt) + recovered = lf_validators.str_to_timestamp(encoded) + assert recovered == dt + + def test_encode_datetime_parseable_by_lfx_str_to_timestamp(self): + from langflow.schema.encoders import encode_datetime + + dt = _utc(2024, 8, 20, 14, 0, 0, 654321) + encoded = encode_datetime(dt) + recovered = lfx_validators.str_to_timestamp(encoded) + assert recovered == dt + + def test_encode_datetime_includes_microseconds(self): + from langflow.schema.encoders import encode_datetime + + dt = _utc(2024, 1, 1, 0, 0, 0, 123456) + result = encode_datetime(dt) + assert "123456" in result diff --git a/src/backend/tests/unit/test_messages.py b/src/backend/tests/unit/test_messages.py index fec270e7b5..1e9b0403c0 100644 --- a/src/backend/tests/unit/test_messages.py +++ b/src/backend/tests/unit/test_messages.py @@ -58,7 +58,7 @@ def test_get_messages(): ] ) limit = 2 - messages = get_messages(sender="User", session_id="session_id2", limit=limit) + messages = get_messages(sender="User", session_id="session_id2", limit=limit, order="ASC") assert len(messages) == limit assert messages[0].text == "Test message 1" assert messages[1].text == "Test message 2" @@ -73,7 +73,7 @@ async def test_aget_messages(): ] ) limit = 2 - messages = await aget_messages(sender="User", session_id="session_id2", limit=limit) + messages = await aget_messages(sender="User", session_id="session_id2", limit=limit, order="ASC") assert len(messages) == limit assert messages[0].text == "Test message 1" assert messages[1].text == "Test message 2" diff --git a/src/backend/tests/unit/test_messages_endpoints.py b/src/backend/tests/unit/test_messages_endpoints.py index aff93dc6c8..3e158a64d8 100644 --- a/src/backend/tests/unit/test_messages_endpoints.py +++ b/src/backend/tests/unit/test_messages_endpoints.py @@ -1,10 +1,10 @@ -from datetime import datetime, timezone from urllib.parse import quote from uuid import UUID, uuid4 import pytest from httpx import AsyncClient from langflow.memory import aadd_messagetables +from langflow.schema.validators import str_to_timestamp, timestamp_to_str # Assuming you have these imports available from langflow.services.auth.utils import get_auth_service @@ -313,8 +313,8 @@ async def test_successfully_update_session_id(client, logged_in_headers, created for message in messages: assert message["session_id"] == new_session_id response_timestamp = message["timestamp"] - timestamp = datetime.strptime(response_timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc) - timestamp_str = timestamp.strftime("%Y-%m-%d %H:%M:%S %Z") + timestamp = str_to_timestamp(response_timestamp) + timestamp_str = timestamp_to_str(timestamp) assert timestamp_str == response_timestamp # Check if the messages ordered by timestamp are in the correct order diff --git a/src/lfx/src/lfx/schema/encoders.py b/src/lfx/src/lfx/schema/encoders.py index 93b6af740d..d45d34abb8 100644 --- a/src/lfx/src/lfx/schema/encoders.py +++ b/src/lfx/src/lfx/schema/encoders.py @@ -7,7 +7,7 @@ def encode_callable(obj: Callable): def encode_datetime(obj: datetime): - return obj.strftime("%Y-%m-%d %H:%M:%S %Z") + return obj.strftime("%Y-%m-%d %H:%M:%S.%f %Z") CUSTOM_ENCODERS = {Callable: encode_callable, datetime: encode_datetime} diff --git a/src/lfx/src/lfx/schema/message.py b/src/lfx/src/lfx/schema/message.py index 678d81739d..571e3b9f9c 100644 --- a/src/lfx/src/lfx/schema/message.py +++ b/src/lfx/src/lfx/schema/message.py @@ -67,7 +67,7 @@ class Message(Data): session_id: str | UUID | None = Field(default="") context_id: str | UUID | None = Field(default="") timestamp: Annotated[str, timestamp_to_str_validator] = Field( - default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z") + default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f %Z") ) flow_id: str | UUID | None = None error: bool = Field(default=False) @@ -116,12 +116,7 @@ class Message(Data): @field_serializer("timestamp") def serialize_timestamp(self, value): - try: - # Try parsing with timezone - return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc) - except ValueError: - # Try parsing without timezone - return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc) + return timestamp_to_str(value) @field_validator("files", mode="before") @classmethod diff --git a/src/lfx/src/lfx/schema/validators.py b/src/lfx/src/lfx/schema/validators.py index 62adbfe2eb..e47b7875bb 100644 --- a/src/lfx/src/lfx/schema/validators.py +++ b/src/lfx/src/lfx/schema/validators.py @@ -3,6 +3,23 @@ from uuid import UUID from pydantic import BeforeValidator +# An ordered list of timestamp formats to try to parse from str, from most to least specific +TIMESTAMP_FORMATS = [ + "%Y-%m-%d %H:%M:%S.%f %Z", # Standard with timezone and microseconds + "%Y-%m-%dT%H:%M:%S.%f%z", # ISO with numeric timezone and microseconds + "%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds + "%Y-%m-%d %H:%M:%S.%f", # Without timezone, with microseconds + "%Y-%m-%d %H:%M:%S %Z", # Standard with timezone + "%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone + "%Y-%m-%dT%H:%M:%S", # ISO format + "%Y-%m-%d %H:%M:%S", # Without timezone +] + +# Formats that carry their own timezone offset — must use astimezone, not replace +_FORMATS_WITH_NUMERIC_TZ = {"%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S%z"} + +TF_WITH_TZ_AND_MICROSECONDS = "%Y-%m-%d %H:%M:%S.%f %Z" + def timestamp_to_str(timestamp: datetime | str) -> str: """Convert timestamp to standardized string format. @@ -13,25 +30,20 @@ def timestamp_to_str(timestamp: datetime | str) -> str: timestamp (datetime | str): Input timestamp either as datetime object or string Returns: - str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS UTC' format + str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS.ffffff UTC' format Raises: ValueError: If string timestamp is in invalid format """ if isinstance(timestamp, str): - # Try parsing with different formats - formats = [ - "%Y-%m-%dT%H:%M:%S", # ISO format - "%Y-%m-%d %H:%M:%S %Z", # Standard with timezone - "%Y-%m-%d %H:%M:%S", # Without timezone - "%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds - "%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone - ] - - for fmt in formats: + for fmt in TIMESTAMP_FORMATS: try: - parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc) - return parsed.strftime("%Y-%m-%d %H:%M:%S %Z") + parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007 + if fmt in _FORMATS_WITH_NUMERIC_TZ: + parsed = parsed.astimezone(timezone.utc) + else: + parsed = parsed.replace(tzinfo=timezone.utc) + return parsed.strftime(TF_WITH_TZ_AND_MICROSECONDS) except ValueError: continue @@ -41,7 +53,7 @@ def timestamp_to_str(timestamp: datetime | str) -> str: # Handle datetime object if timestamp.tzinfo is None: timestamp = timestamp.replace(tzinfo=timezone.utc) - return timestamp.strftime("%Y-%m-%d %H:%M:%S %Z") + return timestamp.strftime(TF_WITH_TZ_AND_MICROSECONDS) def str_to_timestamp(timestamp: str | datetime) -> datetime: @@ -56,14 +68,20 @@ def str_to_timestamp(timestamp: str | datetime) -> datetime: datetime: Datetime object with UTC timezone Raises: - ValueError: If string timestamp is not in 'YYYY-MM-DD HH:MM:SS UTC' format + ValueError: If string timestamp is not in a recognised format """ if isinstance(timestamp, str): - try: - return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc) - except ValueError as e: - msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS UTC" - raise ValueError(msg) from e + for fmt in TIMESTAMP_FORMATS: + try: + parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007 + if fmt in _FORMATS_WITH_NUMERIC_TZ: + return parsed.astimezone(timezone.utc) + return parsed.replace(tzinfo=timezone.utc) + except ValueError: + continue + + msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS.ffffff UTC" + raise ValueError(msg) return timestamp @@ -81,33 +99,7 @@ def timestamp_with_fractional_seconds(timestamp: datetime | str) -> str: Raises: ValueError: If string timestamp is in invalid format """ - if isinstance(timestamp, str): - # Try parsing with different formats - formats = [ - "%Y-%m-%d %H:%M:%S.%f %Z", # Standard with timezone - "%Y-%m-%d %H:%M:%S.%f", # Without timezone - "%Y-%m-%dT%H:%M:%S.%f", # ISO format - "%Y-%m-%dT%H:%M:%S.%f%z", # ISO with numeric timezone - # Also try without fractional seconds - "%Y-%m-%d %H:%M:%S %Z", - "%Y-%m-%d %H:%M:%S", - "%Y-%m-%dT%H:%M:%S", - ] - - for fmt in formats: - try: - parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc) - return parsed.strftime("%Y-%m-%d %H:%M:%S.%f %Z") - except ValueError: - continue - - msg = f"Invalid timestamp format: {timestamp}" - raise ValueError(msg) - - # Handle datetime object - if timestamp.tzinfo is None: - timestamp = timestamp.replace(tzinfo=timezone.utc) - return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f %Z") + return timestamp_to_str(timestamp) timestamp_to_str_validator = BeforeValidator(timestamp_to_str) diff --git a/src/lfx/tests/unit/schema/test_schema_message.py b/src/lfx/tests/unit/schema/test_schema_message.py index 2989e36e9a..05692869e7 100644 --- a/src/lfx/tests/unit/schema/test_schema_message.py +++ b/src/lfx/tests/unit/schema/test_schema_message.py @@ -166,13 +166,17 @@ def test_message_serialization(): # Create a timestamp with timezone message = Message(text="Test message", sender=MESSAGE_SENDER_USER) timestamp_str = message.timestamp - timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc) + timestamp_dt = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc) + serialized = message.model_dump() assert serialized["text"] == "Test message" assert serialized["sender"] == MESSAGE_SENDER_USER - assert serialized["timestamp"] == timestamp - assert serialized["timestamp"].tzinfo == timezone.utc + assert serialized["timestamp"] == timestamp_str + + parsed = datetime.strptime(serialized["timestamp"], "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc) + assert parsed.tzinfo == timezone.utc + assert parsed == timestamp_dt def test_message_to_lc_without_sender(): @@ -188,17 +192,26 @@ def test_timestamp_serialization(): # Test with timezone msg1 = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45 UTC") serialized1 = msg1.model_dump() - assert serialized1["timestamp"].tzinfo == timezone.utc + ts1 = datetime.strptime(serialized1["timestamp"], "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc) + assert ts1.tzinfo == timezone.utc # Test without timezone msg2 = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45") serialized2 = msg2.model_dump() - assert serialized2["timestamp"].tzinfo == timezone.utc + ts2 = datetime.strptime(serialized2["timestamp"], "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc) + assert ts2.tzinfo == timezone.utc # Test that both formats result in equivalent UTC times when appropriate msg_with_tz = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45 UTC") msg_without_tz = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45") - assert msg_with_tz.model_dump()["timestamp"] == msg_without_tz.model_dump()["timestamp"] + + ser_with_tz = msg_with_tz.model_dump()["timestamp"] + ser_without_tz = msg_without_tz.model_dump()["timestamp"] + + parsed_with_tz = datetime.strptime(ser_with_tz, "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc) + parsed_without_tz = datetime.strptime(ser_without_tz, "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc) + + assert parsed_with_tz == parsed_without_tz def test_message_with_image_object_direct():