pengjunjie/langflow
1
0
Fork 1
mirror of https://github.com/langflow-ai/langflow.git synced 2026-07-27 07:00:38 +08:00
Code Issues Packages Projects Releases Wiki Activity

fix: Add microseconds support for PostgreSQL data store (#9941)

Browse Source 
* Add microseconds support for PostgreSQL data store

* Fix UT + Pydantic model

* Extract TZ's as contants

* Return TS in str from serialize_timestamp, because microseconds may be lost on Pydantic conversion (str type)

* Fix inconsistent err message

* Fix UT test_timestamp_serialization

* Reuse timestamp_to_str method

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes

* Code review fixes

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes (attempt 2/3)

* [autofix.ci] apply automated fixes (attempt 3/3)

* fix: address CR feedback on timestamp microsecond support

Fix all critical and minor issues raised in code review:

- timestamp_to_str in lfx: datetime path was missing .%f, dropping
  microseconds entirely (two messages per second got identical timestamps)
- str_to_timestamp in both modules: replace single strptime with a loop
  over all TIMESTAMP_FORMATS restoring backward compat with old strings
  without microseconds ("2024-06-15 10:30:00 UTC")
- .replace(tzinfo=utc) → .astimezone(utc) for formats with numeric tz
  offsets, fixing silent time corruption for non-UTC users
- TF_WITH_TZ_AND_MICROSECONDS_ISO: was a duplicate of
  TF_WITH_TZ_AND_MICROSECONDS; changed to "%Y-%m-%dT%H:%M:%S.%f%z"
- encoders.py encode_datetime: add .%f so output is parseable by
  the validator chain
- playground_events.py PlaygroundEvent and TokenEvent default_factory:
  add .%f to timestamp format strings
- model.py: fix comment .FFFF → .ffffff; use astimezone in fromisoformat
  fallback when parsed datetime carries a tz offset
- Add 35 unit tests covering microsecond preservation, correct UTC
  conversion for non-UTC offsets, roundtrip losslessness, sub-second
  ordering, backward compat and encoder output parseability

* Add `order` parameter to `get_messages` and `aget_messages` tests

* Refactor: use `str_to_timestamp` and `timestamp_to_str` for timestamp parsing in message endpoint tests

* [autofix.ci] apply automated fixes

* [autofix.ci] apply automated fixes (attempt 2/3)

* [autofix.ci] apply automated fixes (attempt 3/3)

* Fix encode_datetime precision

---------

Co-authored-by: autofix-ci[bot] <114827586+autofix-ci[bot]@users.noreply.github.com>
Co-authored-by: ogabrielluiz <gabriel@langflow.org>
Co-authored-by: Gabriel Luiz Freitas Almeida <gabriel@logspace.ai>
This commit is contained in:
dzbanek717
2026-04-28 23:01:41 +02:00
committed by GitHub
parent 9dcdc704ec
commit f7d63d081d
11 changed files with 272 additions and 135 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
src/backend/base/langflow/schema/encoders.py
View File

@ -7,7 +7,7 @@ def encode_callable(obj: Callable):
def encode_datetime(obj: datetime):
return obj.strftime("%Y-%m-%d %H:%M:%S %Z")
return obj.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
CUSTOM_ENCODERS = {Callable: encode_callable, datetime: encode_datetime}

4
src/backend/base/langflow/schema/playground_events.py
View File

@ -21,7 +21,7 @@ class PlaygroundEvent(BaseModel):
files: list[str] | None = Field(default=None)
text: str | None = Field(default=None)
timestamp: Annotated[str, timestamp_to_str_validator] = Field(
default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z")
default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f %Z")
)
id_: UUID | str | None = Field(default=None, alias="id")
@ -80,7 +80,7 @@ class TokenEvent(BaseModel):
chunk: str = Field(...)
id: UUID | str | None = Field(alias="id")
timestamp: Annotated[str, timestamp_to_str_validator] = Field(
default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z")
default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f %Z")
)

95
src/backend/base/langflow/schema/validators.py
View File

@ -2,6 +2,24 @@ from datetime import datetime, timezone
from pydantic import BeforeValidator
TF_WITH_TZ_AND_MICROSECONDS = "%Y-%m-%d %H:%M:%S.%f %Z"
TF_WITH_TZ_AND_MICROSECONDS_ISO = "%Y-%m-%dT%H:%M:%S.%f%z"
# An ordered list of timestamp formats to try to parse from str, from most to least specific
TIMESTAMP_FORMATS = [
TF_WITH_TZ_AND_MICROSECONDS, # Standard with timezone and microseconds
TF_WITH_TZ_AND_MICROSECONDS_ISO, # ISO with numeric timezone and microseconds
"%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds
"%Y-%m-%d %H:%M:%S.%f", # Without timezone, with microseconds
"%Y-%m-%d %H:%M:%S %Z", # Standard with timezone
"%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone
"%Y-%m-%dT%H:%M:%S", # ISO format
"%Y-%m-%d %H:%M:%S", # Without timezone
]
# Formats that carry their own timezone offset — must use astimezone, not replace
_FORMATS_WITH_NUMERIC_TZ = {TF_WITH_TZ_AND_MICROSECONDS_ISO, "%Y-%m-%dT%H:%M:%S%z"}
def timestamp_to_str(timestamp: datetime | str) -> str:
"""Convert timestamp to standardized string format.
@ -12,25 +30,20 @@ def timestamp_to_str(timestamp: datetime | str) -> str:
timestamp (datetime | str): Input timestamp either as datetime object or string
Returns:
str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS UTC' format
str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS.ffffff UTC' format
Raises:
ValueError: If string timestamp is in invalid format
"""
if isinstance(timestamp, str):
# Try parsing with different formats
formats = [
"%Y-%m-%dT%H:%M:%S", # ISO format
"%Y-%m-%d %H:%M:%S %Z", # Standard with timezone
"%Y-%m-%d %H:%M:%S", # Without timezone
"%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds
"%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone
]
for fmt in formats:
for fmt in TIMESTAMP_FORMATS:
try:
parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc)
return parsed.strftime("%Y-%m-%d %H:%M:%S %Z")
parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007
if fmt in _FORMATS_WITH_NUMERIC_TZ:
parsed = parsed.astimezone(timezone.utc)
else:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.strftime(TF_WITH_TZ_AND_MICROSECONDS)
except ValueError:
continue
@ -40,7 +53,7 @@ def timestamp_to_str(timestamp: datetime | str) -> str:
# Handle datetime object
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
return timestamp.strftime("%Y-%m-%d %H:%M:%S %Z")
return timestamp.strftime(TF_WITH_TZ_AND_MICROSECONDS)
def str_to_timestamp(timestamp: str | datetime) -> datetime:
@ -55,60 +68,22 @@ def str_to_timestamp(timestamp: str | datetime) -> datetime:
datetime: Datetime object with UTC timezone
Raises:
ValueError: If string timestamp is not in 'YYYY-MM-DD HH:MM:SS UTC' format
ValueError: If string timestamp is not in a recognised format
"""
if isinstance(timestamp, str):
try:
return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc)
except ValueError as e:
msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS UTC"
raise ValueError(msg) from e
return timestamp
def timestamp_with_fractional_seconds(timestamp: datetime | str) -> str:
"""Convert timestamp to string format including fractional seconds.
Handles multiple input formats and ensures consistent UTC timezone output.
Args:
timestamp (datetime | str): Input timestamp either as datetime object or string
Returns:
str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS.ffffff UTC' format
Raises:
ValueError: If string timestamp is in invalid format
"""
if isinstance(timestamp, str):
# Try parsing with different formats
formats = [
"%Y-%m-%d %H:%M:%S.%f %Z", # Standard with timezone
"%Y-%m-%d %H:%M:%S.%f", # Without timezone
"%Y-%m-%dT%H:%M:%S.%f", # ISO format
"%Y-%m-%dT%H:%M:%S.%f%z", # ISO with numeric timezone
# Also try without fractional seconds
"%Y-%m-%d %H:%M:%S %Z",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%dT%H:%M:%S",
]
for fmt in formats:
for fmt in TIMESTAMP_FORMATS:
try:
parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc)
return parsed.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007
if fmt in _FORMATS_WITH_NUMERIC_TZ:
return parsed.astimezone(timezone.utc)
return parsed.replace(tzinfo=timezone.utc)
except ValueError:
continue
msg = f"Invalid timestamp format: {timestamp}"
msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS.ffffff UTC"
raise ValueError(msg)
# Handle datetime object
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
return timestamp
timestamp_to_str_validator = BeforeValidator(timestamp_to_str)
timestamp_with_fractional_seconds_validator = BeforeValidator(timestamp_with_fractional_seconds)
str_to_timestamp_validator = BeforeValidator(str_to_timestamp)

17
src/backend/base/langflow/services/database/models/message/model.py
View File

@ -10,7 +10,7 @@ from sqlmodel import JSON, Column, Field, SQLModel
from langflow.schema.content_block import ContentBlock
from langflow.schema.properties import Properties
from langflow.schema.validators import str_to_timestamp_validator
from langflow.schema.validators import TF_WITH_TZ_AND_MICROSECONDS, str_to_timestamp, str_to_timestamp_validator
if TYPE_CHECKING:
from langflow.schema.message import Message
@ -39,11 +39,11 @@ class MessageBase(SQLModel):
if isinstance(value, datetime):
if value.tzinfo is None:
value = value.replace(tzinfo=timezone.utc)
return value.strftime("%Y-%m-%d %H:%M:%S %Z")
return value.strftime(TF_WITH_TZ_AND_MICROSECONDS)
if isinstance(value, str):
value = datetime.fromisoformat(value).replace(tzinfo=timezone.utc)
return value.strftime("%Y-%m-%d %H:%M:%S %Z")
dt = str_to_timestamp(value) # unified, UTC-normalized
return dt.strftime(TF_WITH_TZ_AND_MICROSECONDS)
return value
@ -85,10 +85,15 @@ class MessageBase(SQLModel):
message.files = image_paths
if isinstance(message.timestamp, str):
# Convert timestamp string in format "YYYY-MM-DD HH:MM:SS.ffffff UTC" to datetime
try:
timestamp = datetime.strptime(message.timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc)
timestamp = datetime.strptime(message.timestamp, TF_WITH_TZ_AND_MICROSECONDS).replace(
tzinfo=timezone.utc
)
except ValueError:
timestamp = datetime.fromisoformat(message.timestamp).replace(tzinfo=timezone.utc)
# Fallback for ISO format if the above fails; astimezone preserves offset if present
parsed = datetime.fromisoformat(message.timestamp)
timestamp = parsed.astimezone(timezone.utc) if parsed.tzinfo else parsed.replace(tzinfo=timezone.utc)
else:
timestamp = message.timestamp

157
src/backend/tests/unit/schema/test_timestamp_validators.py Normal file
View File

@ -0,0 +1,157 @@
"""Unit tests for timestamp validator functions in both langflow and lfx schemas."""
from datetime import datetime, timezone
import langflow.schema.validators as lf_validators
import lfx.schema.validators as lfx_validators
import pytest
# ---------------------------------------------------------------------------
# Helpers
# ---------------------------------------------------------------------------
def _utc(year, month, day, hour, minute, second, microsecond=0):
return datetime(year, month, day, hour, minute, second, microsecond, tzinfo=timezone.utc)
# ---------------------------------------------------------------------------
# Parametrize over both implementations so every test runs for both
# ---------------------------------------------------------------------------
VALIDATOR_MODULES = [
pytest.param(lf_validators, id="langflow"),
pytest.param(lfx_validators, id="lfx"),
]
@pytest.mark.parametrize("mod", VALIDATOR_MODULES)
class TestTimestampToStr:
"""timestamp_to_str converts various inputs to the canonical string format."""
def test_datetime_with_microseconds_preserved(self, mod):
dt = _utc(2024, 1, 15, 10, 30, 45, 123456)
result = mod.timestamp_to_str(dt)
assert result == "2024-01-15 10:30:45.123456 UTC"
def test_datetime_without_microseconds_zero_padded(self, mod):
dt = _utc(2024, 6, 1, 0, 0, 0, 0)
result = mod.timestamp_to_str(dt)
assert result == "2024-06-01 00:00:00.000000 UTC"
def test_datetime_naive_treated_as_utc(self, mod):
dt = datetime(2024, 3, 10, 8, 0, 0, 500000) # noqa: DTZ001 — intentionally naive to test the fallback
result = mod.timestamp_to_str(dt)
assert result == "2024-03-10 08:00:00.500000 UTC"
def test_string_old_format_no_microseconds(self, mod):
"""Backward compat: old format without microseconds must still parse."""
result = mod.timestamp_to_str("2024-06-15 10:30:00 UTC")
assert result == "2024-06-15 10:30:00.000000 UTC"
def test_string_new_format_with_microseconds(self, mod):
result = mod.timestamp_to_str("2024-01-15 10:30:45.123456 UTC")
assert result == "2024-01-15 10:30:45.123456 UTC"
def test_string_iso_with_numeric_tz_offset_positive(self, mod):
"""Non-UTC numeric offset must be converted correctly, not silently corrupted."""
# +05:00 means real UTC is 05:30:45
result = mod.timestamp_to_str("2024-01-15T10:30:45+05:00")
assert result == "2024-01-15 05:30:45.000000 UTC"
def test_string_iso_with_numeric_tz_offset_negative(self, mod):
# -03:00 means real UTC is 13:30:45
result = mod.timestamp_to_str("2024-01-15T10:30:45-03:00")
assert result == "2024-01-15 13:30:45.000000 UTC"
def test_string_invalid_format_raises(self, mod):
with pytest.raises(ValueError, match="Invalid timestamp format"):
mod.timestamp_to_str("not-a-date")
@pytest.mark.parametrize("mod", VALIDATOR_MODULES)
class TestStrToTimestamp:
"""str_to_timestamp converts strings (and datetimes) back to datetime objects."""
def test_new_format_with_microseconds(self, mod):
result = mod.str_to_timestamp("2024-01-15 10:30:45.123456 UTC")
assert result == _utc(2024, 1, 15, 10, 30, 45, 123456)
def test_old_format_without_microseconds_backward_compat(self, mod):
"""Old-format strings must parse without crashing."""
result = mod.str_to_timestamp("2024-06-15 10:30:00 UTC")
assert result == _utc(2024, 6, 15, 10, 30, 0, 0)
def test_iso_with_numeric_tz_offset(self, mod):
result = mod.str_to_timestamp("2024-01-15T10:30:45+05:00")
assert result == _utc(2024, 1, 15, 5, 30, 45, 0)
def test_passthrough_for_datetime_object(self, mod):
dt = _utc(2024, 5, 20, 12, 0, 0, 999)
assert mod.str_to_timestamp(dt) is dt
def test_invalid_string_raises(self, mod):
with pytest.raises(ValueError, match="Invalid timestamp format"):
mod.str_to_timestamp("not-a-date")
@pytest.mark.parametrize("mod", VALIDATOR_MODULES)
class TestRoundtrip:
"""timestamp_to_str ↔ str_to_timestamp must be lossless for microseconds."""
def test_microseconds_survive_roundtrip(self, mod):
original = _utc(2024, 11, 3, 22, 59, 59, 999999)
as_str = mod.timestamp_to_str(original)
recovered = mod.str_to_timestamp(as_str)
assert recovered == original
def test_zero_microseconds_survive_roundtrip(self, mod):
original = _utc(2024, 1, 1, 0, 0, 0, 0)
as_str = mod.timestamp_to_str(original)
recovered = mod.str_to_timestamp(as_str)
assert recovered == original
@pytest.mark.parametrize("mod", VALIDATOR_MODULES)
class TestMessageOrdering:
"""Two messages differing only by microseconds must maintain correct order."""
def test_sub_second_ordering(self, mod):
earlier = _utc(2024, 6, 15, 12, 0, 0, 1)
later = _utc(2024, 6, 15, 12, 0, 0, 999999)
earlier_str = mod.timestamp_to_str(earlier)
later_str = mod.timestamp_to_str(later)
# String comparison must preserve chronological order
assert earlier_str < later_str
# And roundtrip back to datetime must also preserve order
assert mod.str_to_timestamp(earlier_str) < mod.str_to_timestamp(later_str)
class TestEncoderCompatibility:
"""encode_datetime output must be parseable by both validator chains."""
def test_encode_datetime_parseable_by_langflow_str_to_timestamp(self):
from langflow.schema.encoders import encode_datetime
dt = _utc(2024, 8, 20, 14, 0, 0, 654321)
encoded = encode_datetime(dt)
recovered = lf_validators.str_to_timestamp(encoded)
assert recovered == dt
def test_encode_datetime_parseable_by_lfx_str_to_timestamp(self):
from langflow.schema.encoders import encode_datetime
dt = _utc(2024, 8, 20, 14, 0, 0, 654321)
encoded = encode_datetime(dt)
recovered = lfx_validators.str_to_timestamp(encoded)
assert recovered == dt
def test_encode_datetime_includes_microseconds(self):
from langflow.schema.encoders import encode_datetime
dt = _utc(2024, 1, 1, 0, 0, 0, 123456)
result = encode_datetime(dt)
assert "123456" in result

4
src/backend/tests/unit/test_messages.py
View File

@ -58,7 +58,7 @@ def test_get_messages():
]
)
limit = 2
messages = get_messages(sender="User", session_id="session_id2", limit=limit)
messages = get_messages(sender="User", session_id="session_id2", limit=limit, order="ASC")
assert len(messages) == limit
assert messages[0].text == "Test message 1"
assert messages[1].text == "Test message 2"
@ -73,7 +73,7 @@ async def test_aget_messages():
]
)
limit = 2
messages = await aget_messages(sender="User", session_id="session_id2", limit=limit)
messages = await aget_messages(sender="User", session_id="session_id2", limit=limit, order="ASC")
assert len(messages) == limit
assert messages[0].text == "Test message 1"
assert messages[1].text == "Test message 2"

6
src/backend/tests/unit/test_messages_endpoints.py
View File

@ -1,10 +1,10 @@
from datetime import datetime, timezone
from urllib.parse import quote
from uuid import UUID, uuid4
import pytest
from httpx import AsyncClient
from langflow.memory import aadd_messagetables
from langflow.schema.validators import str_to_timestamp, timestamp_to_str
# Assuming you have these imports available
from langflow.services.auth.utils import get_auth_service
@ -313,8 +313,8 @@ async def test_successfully_update_session_id(client, logged_in_headers, created
for message in messages:
assert message["session_id"] == new_session_id
response_timestamp = message["timestamp"]
timestamp = datetime.strptime(response_timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc)
timestamp_str = timestamp.strftime("%Y-%m-%d %H:%M:%S %Z")
timestamp = str_to_timestamp(response_timestamp)
timestamp_str = timestamp_to_str(timestamp)
assert timestamp_str == response_timestamp
# Check if the messages ordered by timestamp are in the correct order

2
src/lfx/src/lfx/schema/encoders.py
View File

@ -7,7 +7,7 @@ def encode_callable(obj: Callable):
def encode_datetime(obj: datetime):
return obj.strftime("%Y-%m-%d %H:%M:%S %Z")
return obj.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
CUSTOM_ENCODERS = {Callable: encode_callable, datetime: encode_datetime}

9
src/lfx/src/lfx/schema/message.py
View File

@ -67,7 +67,7 @@ class Message(Data):
session_id: str | UUID | None = Field(default="")
context_id: str | UUID | None = Field(default="")
timestamp: Annotated[str, timestamp_to_str_validator] = Field(
default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S %Z")
default_factory=lambda: datetime.now(timezone.utc).strftime("%Y-%m-%d %H:%M:%S.%f %Z")
)
flow_id: str | UUID | None = None
error: bool = Field(default=False)
@ -116,12 +116,7 @@ class Message(Data):
@field_serializer("timestamp")
def serialize_timestamp(self, value):
try:
# Try parsing with timezone
return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc)
except ValueError:
# Try parsing without timezone
return datetime.strptime(value.strip(), "%Y-%m-%d %H:%M:%S").replace(tzinfo=timezone.utc)
return timestamp_to_str(value)
@field_validator("files", mode="before")
@classmethod

86
src/lfx/src/lfx/schema/validators.py
View File

@ -3,6 +3,23 @@ from uuid import UUID
from pydantic import BeforeValidator
# An ordered list of timestamp formats to try to parse from str, from most to least specific
TIMESTAMP_FORMATS = [
"%Y-%m-%d %H:%M:%S.%f %Z", # Standard with timezone and microseconds
"%Y-%m-%dT%H:%M:%S.%f%z", # ISO with numeric timezone and microseconds
"%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds
"%Y-%m-%d %H:%M:%S.%f", # Without timezone, with microseconds
"%Y-%m-%d %H:%M:%S %Z", # Standard with timezone
"%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone
"%Y-%m-%dT%H:%M:%S", # ISO format
"%Y-%m-%d %H:%M:%S", # Without timezone
]
# Formats that carry their own timezone offset — must use astimezone, not replace
_FORMATS_WITH_NUMERIC_TZ = {"%Y-%m-%dT%H:%M:%S.%f%z", "%Y-%m-%dT%H:%M:%S%z"}
TF_WITH_TZ_AND_MICROSECONDS = "%Y-%m-%d %H:%M:%S.%f %Z"
def timestamp_to_str(timestamp: datetime | str) -> str:
"""Convert timestamp to standardized string format.
@ -13,25 +30,20 @@ def timestamp_to_str(timestamp: datetime | str) -> str:
timestamp (datetime | str): Input timestamp either as datetime object or string
Returns:
str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS UTC' format
str: Formatted timestamp string in 'YYYY-MM-DD HH:MM:SS.ffffff UTC' format
Raises:
ValueError: If string timestamp is in invalid format
"""
if isinstance(timestamp, str):
# Try parsing with different formats
formats = [
"%Y-%m-%dT%H:%M:%S", # ISO format
"%Y-%m-%d %H:%M:%S %Z", # Standard with timezone
"%Y-%m-%d %H:%M:%S", # Without timezone
"%Y-%m-%dT%H:%M:%S.%f", # ISO with microseconds
"%Y-%m-%dT%H:%M:%S%z", # ISO with numeric timezone
]
for fmt in formats:
for fmt in TIMESTAMP_FORMATS:
try:
parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc)
return parsed.strftime("%Y-%m-%d %H:%M:%S %Z")
parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007
if fmt in _FORMATS_WITH_NUMERIC_TZ:
parsed = parsed.astimezone(timezone.utc)
else:
parsed = parsed.replace(tzinfo=timezone.utc)
return parsed.strftime(TF_WITH_TZ_AND_MICROSECONDS)
except ValueError:
continue
@ -41,7 +53,7 @@ def timestamp_to_str(timestamp: datetime | str) -> str:
# Handle datetime object
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
return timestamp.strftime("%Y-%m-%d %H:%M:%S %Z")
return timestamp.strftime(TF_WITH_TZ_AND_MICROSECONDS)
def str_to_timestamp(timestamp: str | datetime) -> datetime:
@ -56,14 +68,20 @@ def str_to_timestamp(timestamp: str | datetime) -> datetime:
datetime: Datetime object with UTC timezone
Raises:
ValueError: If string timestamp is not in 'YYYY-MM-DD HH:MM:SS UTC' format
ValueError: If string timestamp is not in a recognised format
"""
if isinstance(timestamp, str):
try:
return datetime.strptime(timestamp, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc)
except ValueError as e:
msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS UTC"
raise ValueError(msg) from e
for fmt in TIMESTAMP_FORMATS:
try:
parsed = datetime.strptime(timestamp.strip(), fmt) # noqa: DTZ007
if fmt in _FORMATS_WITH_NUMERIC_TZ:
return parsed.astimezone(timezone.utc)
return parsed.replace(tzinfo=timezone.utc)
except ValueError:
continue
msg = f"Invalid timestamp format: {timestamp}. Expected format: YYYY-MM-DD HH:MM:SS.ffffff UTC"
raise ValueError(msg)
return timestamp
@ -81,33 +99,7 @@ def timestamp_with_fractional_seconds(timestamp: datetime | str) -> str:
Raises:
ValueError: If string timestamp is in invalid format
"""
if isinstance(timestamp, str):
# Try parsing with different formats
formats = [
"%Y-%m-%d %H:%M:%S.%f %Z", # Standard with timezone
"%Y-%m-%d %H:%M:%S.%f", # Without timezone
"%Y-%m-%dT%H:%M:%S.%f", # ISO format
"%Y-%m-%dT%H:%M:%S.%f%z", # ISO with numeric timezone
# Also try without fractional seconds
"%Y-%m-%d %H:%M:%S %Z",
"%Y-%m-%d %H:%M:%S",
"%Y-%m-%dT%H:%M:%S",
]
for fmt in formats:
try:
parsed = datetime.strptime(timestamp.strip(), fmt).replace(tzinfo=timezone.utc)
return parsed.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
except ValueError:
continue
msg = f"Invalid timestamp format: {timestamp}"
raise ValueError(msg)
# Handle datetime object
if timestamp.tzinfo is None:
timestamp = timestamp.replace(tzinfo=timezone.utc)
return timestamp.strftime("%Y-%m-%d %H:%M:%S.%f %Z")
return timestamp_to_str(timestamp)
timestamp_to_str_validator = BeforeValidator(timestamp_to_str)

25
src/lfx/tests/unit/schema/test_schema_message.py
View File

@ -166,13 +166,17 @@ def test_message_serialization():
# Create a timestamp with timezone
message = Message(text="Test message", sender=MESSAGE_SENDER_USER)
timestamp_str = message.timestamp
timestamp = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S %Z").replace(tzinfo=timezone.utc)
timestamp_dt = datetime.strptime(timestamp_str, "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc)
serialized = message.model_dump()
assert serialized["text"] == "Test message"
assert serialized["sender"] == MESSAGE_SENDER_USER
assert serialized["timestamp"] == timestamp
assert serialized["timestamp"].tzinfo == timezone.utc
assert serialized["timestamp"] == timestamp_str
parsed = datetime.strptime(serialized["timestamp"], "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc)
assert parsed.tzinfo == timezone.utc
assert parsed == timestamp_dt
def test_message_to_lc_without_sender():
@ -188,17 +192,26 @@ def test_timestamp_serialization():
# Test with timezone
msg1 = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45 UTC")
serialized1 = msg1.model_dump()
assert serialized1["timestamp"].tzinfo == timezone.utc
ts1 = datetime.strptime(serialized1["timestamp"], "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc)
assert ts1.tzinfo == timezone.utc
# Test without timezone
msg2 = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45")
serialized2 = msg2.model_dump()
assert serialized2["timestamp"].tzinfo == timezone.utc
ts2 = datetime.strptime(serialized2["timestamp"], "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc)
assert ts2.tzinfo == timezone.utc
# Test that both formats result in equivalent UTC times when appropriate
msg_with_tz = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45 UTC")
msg_without_tz = Message(text="Test message", sender=MESSAGE_SENDER_USER, timestamp="2023-12-25 15:30:45")
assert msg_with_tz.model_dump()["timestamp"] == msg_without_tz.model_dump()["timestamp"]
ser_with_tz = msg_with_tz.model_dump()["timestamp"]
ser_without_tz = msg_without_tz.model_dump()["timestamp"]
parsed_with_tz = datetime.strptime(ser_with_tz, "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc)
parsed_without_tz = datetime.strptime(ser_without_tz, "%Y-%m-%d %H:%M:%S.%f %Z").replace(tzinfo=timezone.utc)
assert parsed_with_tz == parsed_without_tz
def test_message_with_image_object_direct():