Skip to content

Commit 22309b0

Browse files
jpswinskijpswinski
committed
Merge branch 'development' into main
2 parents 16598c1 + d829363 commit 22309b0

File tree

9 files changed

+137
-65
lines changed

9 files changed

+137
-65
lines changed

‎README.md

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -33,7 +33,7 @@ Example usage:
3333
from sliderule import icesat2
3434

3535
# initialize
36-
icesat2.init("slideruleearth.io", verbose=True)
36+
icesat2.init("slideruleearth.io", verbose=False)
3737

3838
# region of interest
3939
region = [ {"lon":-105.82971551223244, "lat": 39.81983728534918},

‎examples/arcticdem.ipynb

Lines changed: 48 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -42,25 +42,57 @@
4242
" \"len\": 20.0,\n",
4343
" \"res\": 10.0,\n",
4444
" \"maxi\": 1,\n",
45-
" \"samples\": [\"arcticdem-mosaic\"] }\n",
45+
" \"samples\": {\"mosaic\": {\"asset\": \"arcticdem-mosaic\", \"radius\": 10.0, \"zonal_stats\": True}} }\n",
4646
"gdf = icesat2.atl06p(parms, asset=asset, resources=[resource])"
4747
]
4848
},
4949
{
5050
"cell_type": "code",
5151
"execution_count": null,
52-
"id": "d0ebba64-93ab-45c8-9c53-cc5078332617",
52+
"id": "fd15bf14-ab10-4cf6-9524-592962a8f8b2",
5353
"metadata": {},
5454
"outputs": [],
5555
"source": [
56-
"gdf[\"delta\"] = gdf[\"h_mean\"] - gdf[\"arcticdem-mosaic-1980-01-06\"]\n",
57-
"gdf[\"delta\"].describe()"
56+
"gdf"
5857
]
5958
},
6059
{
6160
"cell_type": "code",
6261
"execution_count": null,
63-
"id": "db8b0b39-b421-46c6-bb5f-7abc6b1168b7",
62+
"id": "59ea096b-3443-4b9a-b114-e818f143ca45",
63+
"metadata": {},
64+
"outputs": [],
65+
"source": [
66+
"gdf[\"value_delta\"] = gdf[\"h_mean\"] - gdf[\"mosaic.value\"]\n",
67+
"gdf[\"value_delta\"].describe()"
68+
]
69+
},
70+
{
71+
"cell_type": "code",
72+
"execution_count": null,
73+
"id": "f80cd750-9b91-406a-ac6a-dc04937cd811",
74+
"metadata": {},
75+
"outputs": [],
76+
"source": [
77+
"gdf[\"mean_delta\"] = gdf[\"h_mean\"] - gdf[\"mosaic.mean\"]\n",
78+
"gdf[\"mean_delta\"].describe()"
79+
]
80+
},
81+
{
82+
"cell_type": "code",
83+
"execution_count": null,
84+
"id": "4e8fe135-bc15-4b8b-887f-ae16ac81487f",
85+
"metadata": {},
86+
"outputs": [],
87+
"source": [
88+
"gdf[\"median_delta\"] = gdf[\"h_mean\"] - gdf[\"mosaic.median\"]\n",
89+
"gdf[\"median_delta\"].describe()"
90+
]
91+
},
92+
{
93+
"cell_type": "code",
94+
"execution_count": null,
95+
"id": "d6785ed8-cb0d-49cb-9de1-30ac5792884a",
6496
"metadata": {},
6597
"outputs": [],
6698
"source": [
@@ -79,9 +111,13 @@
79111
"legend_elements.append(matplotlib.lines.Line2D([0], [0], color='red', lw=6, label='ATL06-SR'))\n",
80112
"\n",
81113
"# Plot ArcticDEM Elevations\n",
82-
"sc2 = ax.scatter(df.index.values, df[\"arcticdem-mosaic-1980-01-06\"].values, c='blue', s=2.5)\n",
114+
"sc2 = ax.scatter(df.index.values, df[\"mosaic.value\"].values, c='blue', s=2.5)\n",
83115
"legend_elements.append(matplotlib.lines.Line2D([0], [0], color='blue', lw=6, label='ArcticDEM'))\n",
84116
"\n",
117+
"# Plot ArcticDEM Mean Elevations\n",
118+
"sc2 = ax.scatter(df.index.values, df[\"mosaic.value\"].values, c='green', s=2.5)\n",
119+
"legend_elements.append(matplotlib.lines.Line2D([0], [0], color='green', lw=6, label='ArcticDEM'))\n",
120+
"\n",
85121
"# Display Legend\n",
86122
"lgd = ax.legend(handles=legend_elements, loc=3, frameon=True)\n",
87123
"lgd.get_frame().set_alpha(1.0)\n",
@@ -108,8 +144,12 @@
108144
"ax.yaxis.grid(True)\n",
109145
"\n",
110146
"# Plot Deltas\n",
111-
"df = gdf[(gdf['rgt'] == 1160) & (gdf['gt'] == 10) & (gdf['cycle'] == 2)]\n",
112-
"sc1 = ax.scatter(df.index.values, df[\"delta\"].values, c='green', s=2.5)\n",
147+
"df1 = gdf[(gdf['rgt'] == 1160) & (gdf['gt'] == 10) & (gdf['cycle'] == 2)]\n",
148+
"sc1 = ax.scatter(df1.index.values, df1[\"value_delta\"].values, c='blue', s=2.5)\n",
149+
"\n",
150+
"# Plot Deltas\n",
151+
"df2 = gdf[(gdf['rgt'] == 1160) & (gdf['gt'] == 10) & (gdf['cycle'] == 2)]\n",
152+
"sc2 = ax.scatter(df2.index.values, df2[\"mean_delta\"].values, c='green', s=2.5)\n",
113153
"\n",
114154
"# Show Plot\n",
115155
"plt.show()"

‎sliderule/icesat2.py

Lines changed: 45 additions & 26 deletions
Original file line numberDiff line numberDiff line change
@@ -531,9 +531,9 @@ def init (url, verbose=False, max_resources=DEFAULT_MAX_REQUESTED_RESOURCES, log
531531
Parameters
532532
----------
533533
url : str
534-
the IP address or hostname of the SlideRule service (note, there is a special case where the url is provided as a list of strings instead of just a string; when a list is provided, the client hardcodes the set of servers that are used to process requests to the exact set provided; this is used for testing and for local installations and can be ignored by most users)
534+
the IP address or hostname of the SlideRule service (slidereearth.io by default)
535535
verbose : bool
536-
whether or not user level log messages received from SlideRule generate a Python log message (see `sliderule.set_verbose <../user_guide/SlideRule.html#set_verbose>`_)
536+
whether or not user level log messages received from SlideRule generate a Python log message
537537
max_resources : int
538538
the maximum number of resources that are allowed to be processed in a single request
539539
loglevel : int
@@ -585,7 +585,7 @@ def cmr(**kwargs):
585585
Parameters
586586
----------
587587
polygon: list
588-
either a single list of longitude,latitude in counter-clockwise order with first and last point matching, defining region of interest (see `polygons <../user_guide/ICESat-2.html#polygons>`_), or a list of such lists when the region includes more than one polygon
588+
either a single list of longitude,latitude in counter-clockwise order with first and last point matching, defining region of interest (see `polygons </rtd/user_guide/ICESat-2.html#id1>`_), or a list of such lists when the region includes more than one polygon
589589
time_start: str
590590
starting time for query in format ``<year>-<month>-<day>T<hour>:<minute>:<second>Z``
591591
time_end: str
@@ -709,16 +709,16 @@ def atl06 (parm, resource, asset=DEFAULT_ASSET):
709709
Parameters
710710
----------
711711
parms: dict
712-
parameters used to configure ATL06-SR algorithm processing (see `Parameters <../user_guide/ICESat-2.html#parameters>`_)
712+
parameters used to configure ATL06-SR algorithm processing (see `Parameters </rtds/user_guide/ICESat-2.html#parameters>`_)
713713
resource: str
714714
ATL03 HDF5 filename
715715
asset: str
716-
data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_)
716+
data source asset (see `Assets </rtd/user_guide/ICESat-2.html#assets>`_)
717717
718718
Returns
719719
-------
720720
GeoDataFrame
721-
gridded elevations (see `Elevations <../user_guide/ICESat-2.html#elevations>`_)
721+
gridded elevations (see `Elevations </rtd/user_guide/ICESat-2.html#elevations>`_)
722722
'''
723723
return atl06p(parm, asset=asset, resources=[resource])
724724

@@ -741,9 +741,9 @@ def atl06p(parm, asset=DEFAULT_ASSET, version=DEFAULT_ICESAT2_SDP_VERSION, callb
741741
Parameters
742742
----------
743743
parms: dict
744-
parameters used to configure ATL06-SR algorithm processing (see `Parameters <../user_guide/ICESat-2.html#parameters>`_)
744+
parameters used to configure ATL06-SR algorithm processing (see `Parameters </rtd/user_guide/ICESat-2.html#parameters>`_)
745745
asset: str
746-
data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_)
746+
data source asset (see `Assets </rtd/user_guide/ICESat-2.html#assets>`_)
747747
version: str
748748
the version of the ATL03 data to use for processing
749749
callbacks: dictionary
@@ -754,7 +754,7 @@ def atl06p(parm, asset=DEFAULT_ASSET, version=DEFAULT_ICESAT2_SDP_VERSION, callb
754754
Returns
755755
-------
756756
GeoDataFrame
757-
gridded elevations (see `Elevations <../user_guide/ICESat-2.html#elevations>`_)
757+
gridded elevations (see `Elevations </rtd/user_guide/ICESat-2.html#elevations>`_)
758758
759759
Examples
760760
--------
@@ -826,14 +826,33 @@ def atl06p(parm, asset=DEFAULT_ASSET, version=DEFAULT_ICESAT2_SDP_VERSION, callb
826826
# Add Right Pair Track Entry
827827
field_dictionary[field_name]['extent_id'] += rsp['extent_id'] | 0x3,
828828
field_dictionary[field_name][field_name] += data[RIGHT_PAIR],
829-
elif 'rsrec' == rsp['__rectype']:
830-
for sample in rsp["samples"]:
831-
time_str = sliderule.gps2utc(sample["time"])
832-
field_name = parm['samples'][rsp['raster_index']] + "-" + time_str.split(" ")[0].strip()
833-
if field_name not in field_dictionary:
834-
field_dictionary[field_name] = {'extent_id': [], field_name: []}
835-
field_dictionary[field_name]['extent_id'] += rsp['extent_id'],
836-
field_dictionary[field_name][field_name] += sample['value'],
829+
elif 'rsrec' == rsp['__rectype'] or 'zsrec' == rsp['__rectype']:
830+
if rsp["num_samples"] <= 0:
831+
continue
832+
# Get field names and set
833+
sample = rsp["samples"][0]
834+
field_names = list(sample.keys())
835+
field_names.remove("__rectype")
836+
field_set = rsp['key']
837+
as_numpy_array = False
838+
if rsp["num_samples"] > 1:
839+
as_numpy_array = True
840+
# On first time, build empty dictionary for field set associated with raster
841+
if field_set not in field_dictionary:
842+
field_dictionary[field_set] = {'extent_id': []}
843+
for field in field_names:
844+
field_dictionary[field_set][field_set + "." + field] = []
845+
# Populate dictionary for field set
846+
field_dictionary[field_set]['extent_id'] += rsp['extent_id'],
847+
for field in field_names:
848+
if as_numpy_array:
849+
data = []
850+
for s in rsp["samples"]:
851+
data += s[field],
852+
field_dictionary[field_set][field_set + "." + field] += numpy.array(data),
853+
else:
854+
field_dictionary[field_set][field_set + "." + field] += sample[field],
855+
837856
# Build Elevation Columns
838857
if num_elevations > 0:
839858
# Initialize Columns
@@ -887,16 +906,16 @@ def atl03s (parm, resource, asset=DEFAULT_ASSET):
887906
Parameters
888907
----------
889908
parms: dict
890-
parameters used to configure ATL03 subsetting (see `Parameters <../user_guide/ICESat-2.html#parameters>`_)
909+
parameters used to configure ATL03 subsetting (see `Parameters </rtd/user_guide/ICESat-2.html#parameters>`_)
891910
resource: str
892911
ATL03 HDF5 filename
893912
asset: str
894-
data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_)
913+
data source asset (see `Assets </rtd/user_guide/ICESat-2.html#assets>`_)
895914
896915
Returns
897916
-------
898917
GeoDataFrame
899-
ATL03 extents (see `Photon Segments <../user_guide/ICESat-2.html#photon-segments>`_)
918+
ATL03 extents (see `Photon Segments </rtd/user_guide/ICESat-2.html#segmented-photon-data>`_)
900919
'''
901920
return atl03sp(parm, asset=asset, resources=[resource])
902921

@@ -918,9 +937,9 @@ def atl03sp(parm, asset=DEFAULT_ASSET, version=DEFAULT_ICESAT2_SDP_VERSION, call
918937
Parameters
919938
----------
920939
parms: dict
921-
parameters used to configure ATL03 subsetting (see `Parameters <../user_guide/ICESat-2.html#parameters>`_)
940+
parameters used to configure ATL03 subsetting (see `Parameters </rtd/user_guide/ICESat-2.html#parameters>`_)
922941
asset: str
923-
data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_)
942+
data source asset (see `Assets </rtd/user_guide/ICESat-2.html#assets>`_)
924943
version: str
925944
the version of the ATL03 data to return
926945
callbacks: dictionary
@@ -931,7 +950,7 @@ def atl03sp(parm, asset=DEFAULT_ASSET, version=DEFAULT_ICESAT2_SDP_VERSION, call
931950
Returns
932951
-------
933952
GeoDataFrame
934-
ATL03 segments (see `Photon Segments <../user_guide/ICESat-2.html#photon-segments>`_)
953+
ATL03 segments (see `Photon Segments </rtd/user_guide/ICESat-2.html#photon-segments>`_)
935954
'''
936955
try:
937956
tstart = time.perf_counter()
@@ -1112,7 +1131,7 @@ def h5 (dataset, resource, asset=DEFAULT_ASSET, datatype=sliderule.datatypes["DY
11121131
resource: str
11131132
HDF5 filename
11141133
asset: str
1115-
data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_)
1134+
data source asset (see `Assets </rtd/user_guide/ICESat-2.html#assets>`_)
11161135
datatype: int
11171136
the type of data the returned dataset list should be in (datasets that are naturally of a different type undergo a best effort conversion to the specified data type before being returned)
11181137
col: int
@@ -1155,7 +1174,7 @@ def h5p (datasets, resource, asset=DEFAULT_ASSET):
11551174
parallel on the server side, but also shares a file context between the reads so that portions of the file that
11561175
need to be read multiple times do not result in multiple requests to S3.
11571176
1158-
For a full discussion of the data type conversion options, see `h5 <../user_guide/ICESat-2.html#h5>`_.
1177+
For a full discussion of the data type conversion options, see `h5 </rtd/api_reference/icesat2.html#h5>`_.
11591178
11601179
Parameters
11611180
----------
@@ -1164,7 +1183,7 @@ def h5p (datasets, resource, asset=DEFAULT_ASSET):
11641183
resource: str
11651184
HDF5 filename
11661185
asset: str
1167-
data source asset (see `Assets <../user_guide/ICESat-2.html#assets>`_)
1186+
data source asset (see `Assets </rtd/user_guide/ICESat-2.html#assets>`_)
11681187
11691188
Returns
11701189
-------

‎sliderule/ipxapi.py

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -97,7 +97,7 @@ def atl03sp(ipx_region, parm, asset=icesat2.DEFAULT_ASSET):
9797
Returns
9898
-------
9999
list
100-
ATL03 segments (see `Photon Segments <../user_guide/ICESat-2.html#photon-segments>`_)
100+
ATL03 segments (see `Photon Segments <../user_guide/ICESat-2.html#segmented-photon-data>`_)
101101
"""
102102
try:
103103
version = ipx_region.product_version

‎sliderule/sliderule.py

Lines changed: 3 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -422,7 +422,7 @@ def source (api, parm={}, stream=False, callbacks={}, path="/source"):
422422
parm: dict
423423
dictionary of request parameters
424424
stream: bool
425-
whether the request is a **normal** service or a **stream** service (see `De-serialization <./SlideRule.html#de-serialization>`_ for more details)
425+
whether the request is a **normal** service or a **stream** service (see `De-serialization </rtd/user_guide/SlideRule.html#de-serialization>`_ for more details)
426426
callbacks: dict
427427
record type callbacks (advanced use)
428428
path: str
@@ -690,7 +690,8 @@ def authenticate (ps_organization, ps_username=None, ps_password=None):
690690
ps_username = login_credentials[0]
691691
ps_password = login_credentials[2]
692692
except Exception as e:
693-
logger.warning("Failed to retrieve username and password from netrc file: {}".format(e))
693+
if ps_organization != PUBLIC_ORG:
694+
logger.warning("Unable to retrieve username and password from netrc file for machine: {}".format(e))
694695

695696
# authenticate to provisioning system
696697
if ps_username and ps_password:

‎tests/test_arcticdem.py

Lines changed: 29 additions & 4 deletions
Original file line numberDiff line numberDiff line change
@@ -17,7 +17,7 @@ def test_vrt(self, domain, organization):
1717
assert abs(rsps["samples"][0][0]["value"] - 80.713500976562) < 0.001
1818
assert rsps["samples"][0][0]["file"] == '/vsis3/pgc-opendata-dems/arcticdem/mosaics/v3.0/2m/70_09/70_09_2_1_2m_v3.0_reg_dem.tif'
1919

20-
def test_sampler(self, domain, asset, organization):
20+
def test_nearestneighbour(self, domain, asset, organization):
2121
icesat2.init(domain, organization=organization)
2222
resource = "ATL03_20190314093716_11600203_005_01.h5"
2323
region = icesat2.toregion(os.path.join(TESTDIR, "data/dicksonfjord.geojson"))
@@ -29,12 +29,37 @@ def test_sampler(self, domain, asset, organization):
2929
"len": 40.0,
3030
"res": 20.0,
3131
"maxi": 1,
32-
"samples": ["arcticdem-mosaic"] }
32+
"samples": {"mosaic": {"asset": "arcticdem-mosaic"}} }
3333
gdf = icesat2.atl06p(parms, asset=asset, resources=[resource])
3434
assert len(gdf) == 964
35-
assert len(gdf.keys()) == 17
35+
assert len(gdf.keys()) == 18
3636
assert gdf["rgt"][0] == 1160
3737
assert gdf["cycle"][0] == 2
3838
assert gdf['segment_id'].describe()["min"] == 405240
3939
assert gdf['segment_id'].describe()["max"] == 405915
40-
assert abs(gdf["arcticdem-mosaic-1980-01-06"].describe()["min"] - 655.14990234375) < 0.0001
40+
assert abs(gdf["mosaic.value"].describe()["min"] - 655.14990234375) < 0.0001
41+
42+
def test_zonal_stats(self, domain, asset, organization):
43+
icesat2.init(domain, organization=organization)
44+
resource = "ATL03_20190314093716_11600203_005_01.h5"
45+
region = icesat2.toregion(os.path.join(TESTDIR, "data/dicksonfjord.geojson"))
46+
parms = { "poly": region['poly'],
47+
"raster": region['raster'],
48+
"cnf": "atl03_high",
49+
"ats": 20.0,
50+
"cnt": 10,
51+
"len": 40.0,
52+
"res": 20.0,
53+
"maxi": 1,
54+
"samples": {"mosaic": {"asset": "arcticdem-mosaic", "radius": 10.0, "zonal_stats": True}} }
55+
gdf = icesat2.atl06p(parms, asset=asset, resources=[resource])
56+
assert len(gdf) == 964
57+
assert len(gdf.keys()) == 25
58+
assert gdf["rgt"][0] == 1160
59+
assert gdf["cycle"][0] == 2
60+
assert gdf['segment_id'].describe()["min"] == 405240
61+
assert gdf['segment_id'].describe()["max"] == 405915
62+
assert abs(gdf["mosaic.value"].describe()["min"] - 655.14990234375) < 0.0001
63+
assert gdf["mosaic.count"].describe()["max"] == 81
64+
assert gdf["mosaic.stdev"].describe()["count"] == 964
65+
assert gdf["mosaic.time"][0] == 1176076818.0

0 commit comments

Comments
 (0)