Skip to content

Commit 87bdde2

Browse files
ecombelles-csgroupecombelles-csgroup
authored
Merge pull request #135 from RS-PYTHON/feat/rspy-160-implement-basic-data-selection-policies
RSPY-160: Implement basic data selection policies
2 parents 8b8a687 + 8f8d632 commit 87bdde2

File tree

1 file changed

+357
-0
lines changed

1 file changed

+357
-0
lines changed

‎notebooks/sprints/sprint23/160_data_selection_policies.ipynb

Lines changed: 357 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,357 @@
1+
{
2+
"cells": [
3+
{
4+
"cell_type": "markdown",
5+
"id": "ba7132d3-2c1a-4567-8402-8d1c060884a6",
6+
"metadata": {},
7+
"source": [
8+
"# Demo: Catalog searches with CQL2 requests"
9+
]
10+
},
11+
{
12+
"cell_type": "markdown",
13+
"id": "b7f9471a-9e77-44fc-b616-012292a605c5",
14+
"metadata": {},
15+
"source": [
16+
"Demo of work done as part of tickets RSPY-160 and RSPY-656. \n",
17+
"This shows the usage of advanced temporal filters following these specifications: https://pforge-exchange2.astrium.eads.net/confluence/display/COPRS/4.+External+data+selection+policies"
18+
]
19+
},
20+
{
21+
"cell_type": "markdown",
22+
"id": "4e1e1441-cd95-4fea-9313-a2aca4f65715",
23+
"metadata": {},
24+
"source": [
25+
"## 0 - Initialization"
26+
]
27+
},
28+
{
29+
"cell_type": "code",
30+
"execution_count": null,
31+
"id": "aa116d61-9a95-46f7-8b3e-186b217289e5",
32+
"metadata": {},
33+
"outputs": [],
34+
"source": [
35+
"import requests\n",
36+
"import os\n",
37+
"import pprint\n",
38+
"import time\n",
39+
"import pystac\n",
40+
"from pystac import Asset, Collection, Extent, Item, SpatialExtent, TemporalExtent, ItemCollection\n",
41+
"# Init environment before running a demo notebook.\n",
42+
"from resources.utils import *\n",
43+
"\n",
44+
"pp = pprint.PrettyPrinter(indent=2, width=80, sort_dicts=False, compact=True)\n",
45+
"session = requests.Session()\n",
46+
"auxip_client, cadip_client, catalog_client, staging_client = init_demo()\n",
47+
"\n",
48+
"if os.getenv(\"RSPY_LOCAL_MODE\") == \"1\":\n",
49+
" href_cadip = \"http://rs-server-cadip:8000\"\n",
50+
" href_adgs = \"http://rs-server-adgs:8000\"\n",
51+
"else:\n",
52+
" href_cadip = href_adgs = os.environ[\"RSPY_WEBSITE\"]\n",
53+
" session.cookies.set(\"session\", os.environ[\"RSPY_OAUTH2_COOKIE\"])\n",
54+
"\n",
55+
"cadip_collection_id = \"cadip_sentinel1\"\n",
56+
"adgs_collection_id = \"adgs\"\n",
57+
"TIMEOUT = 10\n",
58+
"collection_description = Collection(\n",
59+
" id=TEST_COLLECTION,\n",
60+
" description=None, # rs-client will provide a default description for us\n",
61+
" extent=Extent(\n",
62+
" spatial=SpatialExtent(bboxes=[-180.0, -90.0, 180.0, 90.0]),\n",
63+
" temporal=TemporalExtent([start_date, stop_date]),\n",
64+
" ),\n",
65+
")\n",
66+
"\n",
67+
"# Init the dask cluster\n",
68+
"from resources.dask_utils import *\n",
69+
"init_dask_cluster_staging(scale=2)\n",
70+
"\n",
71+
"# Reload the global vars again\n",
72+
"from resources.dask_utils import *\n",
73+
"\n",
74+
"# Use the staging cluster\n",
75+
"dask_gateway = dask_gateway_staging\n",
76+
"dask_cluster = dask_cluster_staging"
77+
]
78+
},
79+
{
80+
"cell_type": "markdown",
81+
"id": "4fd1857f-fb1c-4a8f-8500-2a3bf52d0dbb",
82+
"metadata": {},
83+
"source": [
84+
"## 1 - Building catalog\n",
85+
"\n",
86+
"The following code is taken from demo called \"404_582_rsclient.ipynb\". \n",
87+
"This section creates a catalog by staging data from AUXIP and CADIP stations."
88+
]
89+
},
90+
{
91+
"cell_type": "code",
92+
"execution_count": null,
93+
"id": "63d8fbdf-90f0-47a1-9c08-5ae3e08245c3",
94+
"metadata": {},
95+
"outputs": [],
96+
"source": [
97+
"# Create a test collection \n",
98+
"collection = create_test_collection()\n",
99+
"\n",
100+
"# Get all the items from the collection \"cadip_sentinel1\" found in the configuration of the CADIP station\n",
101+
"items_collection_cadip = list(cadip_client.get_items(cadip_collection_id))\n",
102+
"assert len(items_collection_cadip) > 0\n",
103+
"\n",
104+
"# Request 14 items from the collection \"adgs\" found in the configuration of the ADGS station\n",
105+
"items_collection_adgs = auxip_client.search(max_items = 14, collections = [adgs_collection_id])\n",
106+
"assert len(items_collection_adgs) == 14\n",
107+
"\n",
108+
"# Starting 2 staging processes, one from the CADIP station and one from the ADGS station\n",
109+
"staging_resp_list = []\n",
110+
"for items in [pystac.ItemCollection(list(items_collection_cadip)), pystac.ItemCollection(list(items_collection_adgs))]:\n",
111+
" staging_resp_list.append(staging_client.run_staging(items.to_dict(), TEST_COLLECTION))\n",
112+
" \n",
113+
"timeout = 120\n",
114+
"\n",
115+
"for resp in staging_resp_list:\n",
116+
" while timeout > 0:\n",
117+
" if \"running\" not in resp[\"status\"]:\n",
118+
" break\n",
119+
" job_info = staging_client.get_job_info(resp[\"jobID\"])\n",
120+
" # pprint.PrettyPrinter(indent=4).pprint(job_info)\n",
121+
" # print(\"\\n\")\n",
122+
" if \"successful\" in job_info[\"status\"]:\n",
123+
" print(\" ----- Job COMPLETED \\n\")\n",
124+
" break\n",
125+
" if \"failed\" in job_info[\"status\"]:\n",
126+
" print(\"-----Job FAILED \\n\")\n",
127+
" break\n",
128+
" time.sleep(2)\n",
129+
" timeout -= 2"
130+
]
131+
},
132+
{
133+
"cell_type": "code",
134+
"execution_count": null,
135+
"id": "c6f7eb0a-b515-42eb-8924-db1a3fc71ceb",
136+
"metadata": {},
137+
"outputs": [],
138+
"source": [
139+
"# Check the catalog for my_test_collection\n",
140+
"result = list(catalog_client.get_items(TEST_COLLECTION))\n",
141+
"\n",
142+
"for item in result:\n",
143+
" print(f\"Item {item.id} has {len(item.assets)} assets\")"
144+
]
145+
},
146+
{
147+
"cell_type": "markdown",
148+
"id": "e5df6927-46d0-459c-9eb7-5b5940385f8b",
149+
"metadata": {},
150+
"source": [
151+
"## 2 - Run various search requests with different filters to retrieve parts of the data\n",
152+
"\n",
153+
"Filters are the ones described here: https://pforge-exchange2.astrium.eads.net/confluence/display/COPRS/4.+External+data+selection+policies\n",
154+
"\n",
155+
"Forked version of Pygeofilter: https://github.com/RS-PYTHON/pygeofilter"
156+
]
157+
},
158+
{
159+
"cell_type": "code",
160+
"execution_count": null,
161+
"id": "33f2cf96-375d-4f9b-b914-07da4099f5d8",
162+
"metadata": {},
163+
"outputs": [],
164+
"source": [
165+
"# ValCover filter\n",
166+
"# This mode gets all files that cover entirely time interval [t0 – dt0, t1 + dt1].\n",
167+
"\n",
168+
"valcover_filter = {\n",
169+
" \"op\": \"t_contains\",\n",
170+
" \"args\": [\n",
171+
" {\"interval\": [{\"property\": \"start_datetime\"}, {\"property\": \"end_datetime\"}]},\n",
172+
" {\"interval\": [\"2024-05-27T09:44:12.509000Z\", \"2024-05-27T09:44:13.509000Z\"]}\n",
173+
" ]\n",
174+
"}\n",
175+
"\n",
176+
"# http://localhost:8003/catalog/search?collections=ecombelles_my_test_collection&filter=T_CONTAINS(INTERVAL(start_datetime,end_datetime),INTERVAL(TIMESTAMP(%272024-05-27T09:44:12.509000Z%27),TIMESTAMP(%272024-05-27T09:44:13.509000Z%27)))\n",
177+
"\n",
178+
"params = {\n",
179+
" \"owner_id\": OWNER_ID,\n",
180+
" \"max_items\": 100,\n",
181+
" \"collections\": [\"my_test_collection\"],\n",
182+
" \"stac_filter\": valcover_filter\n",
183+
"}\n",
184+
"\n",
185+
"catalog_client.search(**params)"
186+
]
187+
},
188+
{
189+
"cell_type": "code",
190+
"execution_count": null,
191+
"id": "1f7b4f30-2830-48fe-996f-bdf79d9459e2",
192+
"metadata": {},
193+
"outputs": [],
194+
"source": [
195+
"# LatestValCover filter\n",
196+
"# This mode gets the latest file that covers entirely time interval [t0 – dt0, t1 + dt1]. The latest record is the one with the more recent Generation Date.\n",
197+
"\n",
198+
"latestvalcover_filter = {\n",
199+
" \"op\": \"t_contains\",\n",
200+
" \"args\": [\n",
201+
" {\"interval\": [{\"property\": \"start_datetime\"}, {\"property\": \"end_datetime\"}]},\n",
202+
" {\"interval\": [\"2024-05-27T09:44:12.509000Z\", \"2024-05-27T09:44:13.509000Z\"]}\n",
203+
" ]\n",
204+
"}\n",
205+
"\n",
206+
"# http://localhost:8003/catalog/search?collections=ecombelles_my_test_collection&filter=T_CONTAINS(INTERVAL(start_datetime,end_datetime),INTERVAL(TIMESTAMP(%272024-05-27T09:44:12.509000Z%27),TIMESTAMP(%272024-05-27T09:44:13.509000Z%27)))&sortby=-properties.created&limit=1\n",
207+
"\n",
208+
"params = {\n",
209+
" \"owner_id\": OWNER_ID,\n",
210+
" \"collections\": [\"my_test_collection\"],\n",
211+
" \"stac_filter\": latestvalcover_filter,\n",
212+
" \"sortby\": [\n",
213+
" {\n",
214+
" \"field\": \"created\",\n",
215+
" \"direction\": \"desc\"\n",
216+
" }\n",
217+
" ],\n",
218+
" \"max_items\": 1,\n",
219+
"}\n",
220+
"\n",
221+
"catalog_client.search(**params)"
222+
]
223+
},
224+
{
225+
"cell_type": "code",
226+
"execution_count": null,
227+
"id": "63aef7db-7545-4468-a7f9-485adc64bf80",
228+
"metadata": {},
229+
"outputs": [],
230+
"source": [
231+
"# ValIntersect filter\n",
232+
"# This mode gets all files that cover partly time interval [t0 – dt0, t1 + dt1].\n",
233+
"\n",
234+
"valintersect_filter = {\n",
235+
" \"op\": \"t_intersects\",\n",
236+
" \"args\": [\n",
237+
" {\"interval\": [{\"property\": \"start_datetime\"}, {\"property\": \"end_datetime\"}]},\n",
238+
" {\"interval\": [\"2024-01-21T09:44:12.509000Z\", \"2024-06-26T09:44:13.509000Z\"]}\n",
239+
" ]\n",
240+
"}\n",
241+
"\n",
242+
"# http://localhost:8003/catalog/search?collections=ecombelles_my_test_collection&filter=T_INTERSECTS(INTERVAL(start_datetime,end_datetime),INTERVAL(TIMESTAMP(%272024-01-21T09:44:12.509000Z%27),TIMESTAMP(%272024-06-26T09:44:13.509000Z%27)))\n",
243+
"\n",
244+
"params = {\n",
245+
" \"owner_id\": OWNER_ID,\n",
246+
" \"max_items\": 100,\n",
247+
" \"collections\": [\"my_test_collection\"],\n",
248+
" \"stac_filter\": valintersect_filter\n",
249+
"}\n",
250+
"\n",
251+
"catalog_client.search(**params)"
252+
]
253+
},
254+
{
255+
"cell_type": "code",
256+
"execution_count": null,
257+
"id": "8fa89d98-58d6-4eaa-bd5b-47297efc0376",
258+
"metadata": {},
259+
"outputs": [],
260+
"source": [
261+
"# LatestValIntersect filter\n",
262+
"# This mode gets the latest file that covers partly time interval [t0 – dt0 , t1 + dt1]. The latest record is the one with the more recent Generation Date.\n",
263+
"\n",
264+
"latestvalintersect_filter = {\n",
265+
" \"op\": \"t_intersects\",\n",
266+
" \"args\": [\n",
267+
" {\"interval\": [{\"property\": \"start_datetime\"}, {\"property\": \"end_datetime\"}]},\n",
268+
" {\"interval\": [\"2024-01-21T09:44:12.509000Z\", \"2024-06-26T09:44:13.509000Z\"]}\n",
269+
" ]\n",
270+
"}\n",
271+
"\n",
272+
"# http://localhost:8003/catalog/search?collections=ecombelles_my_test_collection&filter=T_INTERSECTS(INTERVAL(start_datetime,end_datetime),INTERVAL(TIMESTAMP(%272024-01-21T09:44:12.509000Z%27),TIMESTAMP(%272024-06-26T09:44:13.509000Z%27)))&sortby=-properties.created&limit=1\n",
273+
"\n",
274+
"params = {\n",
275+
" \"owner_id\": OWNER_ID,\n",
276+
" \"collections\": [\"my_test_collection\"],\n",
277+
" \"stac_filter\": latestvalintersect_filter,\n",
278+
" \"sortby\": [{\"field\": \"created\", \"direction\": \"desc\"}],\n",
279+
" \"max_items\": 1,\n",
280+
"}\n",
281+
"\n",
282+
"catalog_client.search(**params)"
283+
]
284+
},
285+
{
286+
"cell_type": "code",
287+
"execution_count": null,
288+
"id": "9d74a444-ef6f-4fa0-b620-53b5785392b9",
289+
"metadata": {},
290+
"outputs": [],
291+
"source": [
292+
"# LatestValidity filter\n",
293+
"# This mode gets a product with the latest Validity Start Time.\n",
294+
"\n",
295+
"# http://localhost:8003/catalog/search?collections=ecombelles_my_test_collection&sortby=-properties.created&limit=1\n",
296+
"\n",
297+
"params = {\n",
298+
" \"owner_id\": OWNER_ID,\n",
299+
" \"collections\": [\"my_test_collection\"],\n",
300+
" \"sortby\": [{\"field\": \"created\", \"direction\": \"desc\"}],\n",
301+
" \"max_items\": 1,\n",
302+
"}\n",
303+
"\n",
304+
"catalog_client.search(**params)"
305+
]
306+
},
307+
{
308+
"cell_type": "markdown",
309+
"id": "9d322ba6-6bb4-4b24-a99e-232fb3db7aa5",
310+
"metadata": {},
311+
"source": [
312+
"## 3 - Delete the catalog collection"
313+
]
314+
},
315+
{
316+
"cell_type": "code",
317+
"execution_count": null,
318+
"id": "9336ea12-c162-4251-8e76-29646cd21cab",
319+
"metadata": {},
320+
"outputs": [],
321+
"source": [
322+
"result = catalog_client.remove_collection(TEST_COLLECTION)\n",
323+
"assert result.json()[\"deleted collection\"] == TEST_COLLECTION\n",
324+
"pp.pprint(result.json())"
325+
]
326+
},
327+
{
328+
"cell_type": "code",
329+
"execution_count": null,
330+
"id": "92076a71-5585-443a-8047-30538ec58e27",
331+
"metadata": {},
332+
"outputs": [],
333+
"source": []
334+
}
335+
],
336+
"metadata": {
337+
"kernelspec": {
338+
"display_name": "Python 3 (ipykernel)",
339+
"language": "python",
340+
"name": "python3"
341+
},
342+
"language_info": {
343+
"codemirror_mode": {
344+
"name": "ipython",
345+
"version": 3
346+
},
347+
"file_extension": ".py",
348+
"mimetype": "text/x-python",
349+
"name": "python",
350+
"nbconvert_exporter": "python",
351+
"pygments_lexer": "ipython3",
352+
"version": "3.11.7"
353+
}
354+
},
355+
"nbformat": 4,
356+
"nbformat_minor": 5
357+
}

0 commit comments

Comments
 (0)