Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Extending temporal search #182

Open
wants to merge 33 commits into
base: main
Choose a base branch
from
Open

Extending temporal search #182

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
33 commits
Select commit Hold shift + click to select a range
2f8d666
Extending datetime search to include start_datetime and end_datetime.
rhysrevans3 Jan 5, 2024
7b33232
Removing _return_date bounding constants.
rhysrevans3 Jan 5, 2024
0a6616e
Setting elasticsearch hostname for dockercompose.
rhysrevans3 Jan 8, 2024
53d44ce
Adding extra test for start_datetime/end_datetime intersecting searches.
rhysrevans3 Jan 8, 2024
18d4be1
Fixing broken tests.
rhysrevans3 Jan 8, 2024
7eddfee
Updating apply_datetime_filter doc string.
rhysrevans3 Jan 8, 2024
65ebbea
Adding changes to change log.
rhysrevans3 Jan 8, 2024
b514f19
Merge branch 'main' into extend_datetime_search
jonhealy1 Feb 1, 2024
5143f00
Adding extra tests.
rhysrevans3 Feb 13, 2024
03c9701
Merge branch 'extend_datetime_search' of github.com:rhysrevans3/stac-…
rhysrevans3 Apr 9, 2024
7595509
Merge branch 'main' of github.com:stac-utils/stac-fastapi-elasticsear…
rhysrevans3 Apr 9, 2024
8ed0928
Adding datetime extension to opensearch.
rhysrevans3 Apr 9, 2024
0a6b9f1
Merge branch main
rhysrevans3 May 10, 2024
5212768
Remove unneeded required install.
rhysrevans3 May 13, 2024
0c1e08f
Update stac_pydantic requirement.
rhysrevans3 May 22, 2024
a2ec61e
Merge branch 'main' of github.com:stac-utils/stac-fastapi-elasticsear…
rhysrevans3 May 22, 2024
c69c9e5
Fixing datetime type issue.
rhysrevans3 May 23, 2024
705bcb1
Fix for datetime format.
rhysrevans3 May 23, 2024
d15173e
Pinning specific version of stac pydantic.
rhysrevans3 May 24, 2024
4f31ac7
Adding missing .git.
rhysrevans3 May 24, 2024
29ea9ba
Switch to https.
rhysrevans3 May 24, 2024
44fd039
Removing stac pydantic requirements.
rhysrevans3 May 24, 2024
ea7646c
Merge branch 'main' of github.com:stac-utils/stac-fastapi-elasticsear…
rhysrevans3 Jun 3, 2024
d003db1
Using deepcopy for multiple item creation.
rhysrevans3 Jun 3, 2024
0d89769
Fixing dt format error.
rhysrevans3 Jun 3, 2024
17818d3
Merge branch 'main' into extend_datetime_search
rhysrevans3 Jun 11, 2024
fb41c03
Merge branch 'main' into extend_datetime_search
rhysrevans3 Jun 18, 2024
8738bdc
Merge branch 'main' into extend_datetime_search
jonhealy1 Oct 6, 2024
4cf5f47
Adding start_datetime and end_datetime to test item.
rhysrevans3 Jan 6, 2025
c99fda5
Merge branch 'main' into extend_datetime_search
rhysrevans3 Jan 6, 2025
b95291e
Merge branch 'main' of github.com:rhysrevans3/stac-fastapi-elasticsea…
rhysrevans3 Feb 6, 2025
14b5097
Merge branch 'extend_datetime_search' of github.com:rhysrevans3/stac-…
rhysrevans3 Feb 6, 2025
2c8ae68
Merge branch 'main' into extend_datetime_search
rhysrevans3 Feb 11, 2025
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -112,6 +112,7 @@ and this project adheres to [Semantic Versioning](http://semver.org/spec/v2.0.0.

- Symlinks from project-specific readme files to main readme [#250](https://github.com/stac-utils/stac-fastapi-elasticsearch-opensearch/pull/250)
- Support for Python 3.12 [#234](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/234)
- Extended Datetime Search to search on start_datetime and end_datetime as well as datetime fields. [#182](https://github.com/stac-utils/stac-fastapi-elasticsearch/pull/182)

### Changed

Expand Down
2 changes: 1 addition & 1 deletion stac_fastapi/core/stac_fastapi/core/core.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -725,7 +725,7 @@ async def update_item(
"""
item = item.model_dump(mode="json")
base_url = str(kwargs["request"].base_url)
now = datetime_type.now(timezone.utc).isoformat().replace("+00:00", "Z")
now = datetime_type.now(timezone.utc).strftime("%Y-%m-%dT%H:%M:%SZ")
item["properties"]["updated"] = now

await self.database.check_collection_exists(collection_id)
Expand Down
110 changes: 101 additions & 9 deletions stac_fastapi/elasticsearch/stac_fastapi/elasticsearch/database_logic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -486,8 +486,8 @@ def apply_collections_filter(search: Search, collection_ids: List[str]):
return search.filter("terms", collection=collection_ids)

@staticmethod
def apply_datetime_filter(search: Search, datetime_search):
"""Apply a filter to search based on datetime field.
def apply_datetime_filter(search: Search, datetime_search: dict):
"""Apply a filter to search on datetime, start_datetime, and end_datetime fields.

Args:
search (Search): The search object to filter.
Expand All @@ -496,17 +496,109 @@ def apply_datetime_filter(search: Search, datetime_search):
Returns:
Search: The filtered search object.
"""
should = []

# If the request is a single datetime return
# items with datetimes equal to the requested datetime OR
# the requested datetime is between their start and end datetimes
if "eq" in datetime_search:
search = search.filter(
"term", **{"properties__datetime": datetime_search["eq"]}
should.extend(
[
Q(
"bool",
filter=[
Q(
"term",
properties__datetime=datetime_search["eq"],
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__start_datetime={
"lte": datetime_search["eq"],
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be "gte": datetime_search["eq"], to find the dates larger than the start_datetime?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes and no, It's a bit counter intuitive. The check it's actually the opposite, so it's searching for any items with a start_datetime that's before or equal to (lte) the given datetime_search["eq"].

},
),
Q(
"range",
properties__end_datetime={
"gte": datetime_search["eq"],
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Should this be "lte": datetime_search["eq"], to find the dates smaller than the end_datetime?

Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Similarly this is searching for any items with a end_datetime that's after or equal to (gte) the given date time datetime_search["eq"].

},
),
],
),
]
)

# If the request is a date range return
# items with datetimes within the requested date range OR
# their startdatetime ithin the requested date range OR
# their enddatetime ithin the requested date range OR
# the requested daterange within their start and end datetimes
else:
search = search.filter(
"range", properties__datetime={"lte": datetime_search["lte"]}
)
search = search.filter(
"range", properties__datetime={"gte": datetime_search["gte"]}
should.extend(
[
Q(
"bool",
filter=[
Q(
"range",
properties__datetime={
"gte": datetime_search["gte"],
"lte": datetime_search["lte"],
},
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__start_datetime={
"gte": datetime_search["gte"],
"lte": datetime_search["lte"],
},
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__end_datetime={
"gte": datetime_search["gte"],
"lte": datetime_search["lte"],
},
),
],
),
Q(
Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Could you explain this last query?

Do both start_ and end_datetimes need to be within the date range?

Copy link
Contributor Author

@rhysrevans3 rhysrevans3 Feb 14, 2025 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

This part concerns date range searches. It returns any items who's datetime property is with the queried date range or who's date range (start_datetime and end_datetime) has any overlap with the queried date range.

  1. The first part searches for items with a datetime property within the queried date range.
  2. The second part searches for an overlap of date ranges and is split in 3:
    i. The start_datetime of an item is within the queried date range.
    ii. The end_datetime of an item is within the queried date range.
    iii. The the queried date range is within the item's date range. The queried start_datetime is after the item's start_datetime and The queried end_datetime is before the item's end_datetime.

I think this captures all cases.

Apologies if any of this is unclear.

"bool",
filter=[
Q(
"range",
properties__start_datetime={
"lte": datetime_search["gte"]
},
),
Q(
"range",
properties__end_datetime={
"gte": datetime_search["lte"]
},
),
],
),
]
)

search = search.query(Q("bool", filter=[Q("bool", should=should)]))

return search

@staticmethod
Expand Down
108 changes: 100 additions & 8 deletions stac_fastapi/opensearch/stac_fastapi/opensearch/database_logic.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -527,7 +527,7 @@ def apply_free_text_filter(search: Search, free_text_queries: Optional[List[str]

@staticmethod
def apply_datetime_filter(search: Search, datetime_search):
"""Apply a filter to search based on datetime field.
"""Apply a filter to search based on datetime field, start_datetime, and end_datetime fields.

Args:
search (Search): The search object to filter.
Expand All @@ -536,17 +536,109 @@ def apply_datetime_filter(search: Search, datetime_search):
Returns:
Search: The filtered search object.
"""
should = []

# If the request is a single datetime return
# items with datetimes equal to the requested datetime OR
# the requested datetime is between their start and end datetimes
if "eq" in datetime_search:
search = search.filter(
"term", **{"properties__datetime": datetime_search["eq"]}
should.extend(
[
Q(
"bool",
filter=[
Q(
"term",
properties__datetime=datetime_search["eq"],
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__start_datetime={
"lte": datetime_search["eq"],
},
),
Q(
"range",
properties__end_datetime={
"gte": datetime_search["eq"],
},
),
],
),
]
)

# If the request is a date range return
# items with datetimes within the requested date range OR
# their startdatetime ithin the requested date range OR
# their enddatetime ithin the requested date range OR
# the requested daterange within their start and end datetimes
else:
search = search.filter(
"range", properties__datetime={"lte": datetime_search["lte"]}
)
search = search.filter(
"range", properties__datetime={"gte": datetime_search["gte"]}
should.extend(
[
Q(
"bool",
filter=[
Q(
"range",
properties__datetime={
"gte": datetime_search["gte"],
"lte": datetime_search["lte"],
},
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__start_datetime={
"gte": datetime_search["gte"],
"lte": datetime_search["lte"],
},
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__end_datetime={
"gte": datetime_search["gte"],
"lte": datetime_search["lte"],
},
),
],
),
Q(
"bool",
filter=[
Q(
"range",
properties__start_datetime={
"lte": datetime_search["gte"]
},
),
Q(
"range",
properties__end_datetime={
"gte": datetime_search["lte"]
},
),
],
),
]
)

search = search.query(Q("bool", filter=[Q("bool", should=should)]))

return search

@staticmethod
Expand Down
Loading