Skip to content

Commit 50c6ee8

Browse files
author
Jon Duckworth
authored
Merge pull request #361 from duckontheweb/windows-ci-fixes
Fix Windows CI builds
2 parents 4658999 + d23be0a commit 50c6ee8

23 files changed

+495
-272
lines changed

‎.github/workflows/continuous-integration.yml

Lines changed: 75 additions & 9 deletions
Original file line numberDiff line numberDiff line change
@@ -7,15 +7,22 @@ on:
77
pull_request:
88

99
jobs:
10-
build:
11-
name: build
10+
test:
11+
name: test
1212
runs-on: ${{ matrix.os }}
1313
strategy:
1414
# Allow other matrix jobs to complete if 1 fails
1515
fail-fast: false
1616
matrix:
17-
python-version: ["3.6", "3.7", "3.8"]
18-
os: [ubuntu-latest, windows-latest, macos-latest]
17+
python-version:
18+
- "3.6"
19+
- "3.7"
20+
- "3.8"
21+
os:
22+
- ubuntu-latest
23+
- windows-latest
24+
- macos-latest
25+
1926
steps:
2027
- uses: actions/checkout@v2
2128

@@ -24,15 +31,38 @@ jobs:
2431
with:
2532
python-version: ${{ matrix.python-version }}
2633

27-
- name: Cache dependencies
34+
- name: Cache dependencies (Linux)
35+
if: startsWith(runner.os, 'Linux')
2836
uses: actions/cache@v2
2937
with:
3038
path: ~/.cache/pip
3139
# Cache based on OS, Python version, and dependency hash
32-
key: pip-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-dev.txt') }}
40+
key: test-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
41+
42+
- name: Cache dependencies (macOS)
43+
if: startsWith(runner.os, 'macOS')
44+
uses: actions/cache@v2
45+
with:
46+
path: ~/Library/Caches/pip
47+
# Cache based on OS, Python version, and dependency hash
48+
key: test-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
49+
50+
- name: Cache dependencies (Windows)
51+
if: startsWith(runner.os, 'Windows')
52+
uses: actions/cache@v2
53+
with:
54+
path: ~\AppData\Local\pip\Cache
55+
# Cache based on OS, Python version, and dependency hash
56+
key: pip-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
57+
58+
- name: Install dependencies
59+
run: |
60+
pip install -r requirements-test.txt
61+
pip install -e ".[validation]"
3362
34-
- name: Execute linters and test suites
35-
run: ./scripts/cibuild
63+
- name: Execute test suite
64+
run: ./scripts/test
65+
shell: bash
3666

3767
- name: Upload All coverage to Codecov
3868
uses: codecov/codecov-action@v1
@@ -42,14 +72,50 @@ jobs:
4272
token: ${{ secrets.CODECOV_TOKEN }}
4373
file: ./coverage.xml
4474
fail_ci_if_error: false
45-
vanilla:
75+
lint:
4676
runs-on: ubuntu-latest
77+
strategy:
78+
# Allow other matrix jobs to complete if 1 fails
79+
fail-fast: false
80+
matrix:
81+
python-version:
82+
- "3.6"
83+
- "3.7"
84+
- "3.8"
85+
4786
steps:
4887
- uses: actions/checkout@v2
88+
89+
- name: Set up Python ${{ matrix.python-version }}
90+
uses: actions/setup-python@v2
91+
with:
92+
python-version: ${{ matrix.python-version }}
93+
94+
- name: Cache dependencies
95+
uses: actions/cache@v2
96+
with:
97+
path: ~/.cache/pip
98+
# Cache based on OS, Python version, and dependency hash
99+
key: lint-${{ runner.os }}-python${{ matrix.python-version }}-${{ hashFiles('requirements-test.txt') }}
100+
101+
- name: Install dependencies
102+
run: |
103+
pip install -r requirements-test.txt
104+
105+
- name: Execute linters & type checkers
106+
run: ./scripts/lint
107+
108+
vanilla:
109+
runs-on: ubuntu-latest
110+
steps:
111+
- uses: actions/checkout@v2
112+
49113
- uses: actions/setup-python@v2
50114
with:
51115
python-version: "3.8"
116+
52117
- name: Install without orjson
53118
run: pip install '.[validation]'
119+
54120
- name: Run unittests
55121
run: python -m unittest discover tests

‎.gitignore

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -1,3 +1,4 @@
1+
tmp*
12
*.pyc
23
*.egg-info
34
build/

‎pystac/layout.py

Lines changed: 37 additions & 13 deletions
Original file line numberDiff line numberDiff line change
@@ -1,10 +1,10 @@
11
from abc import abstractmethod, ABC
22
from collections import OrderedDict
3-
import os
43
from string import Formatter
54
from typing import Any, Callable, Dict, List, Optional, TYPE_CHECKING, Union
65

76
import pystac
7+
from pystac.utils import safe_urlparse, join_path_or_url, JoinType
88

99
if TYPE_CHECKING:
1010
from pystac.stac_object import STACObject as STACObject_Type
@@ -374,36 +374,51 @@ def __init__(
374374
def get_catalog_href(
375375
self, cat: "Catalog_Type", parent_dir: str, is_root: bool
376376
) -> str:
377+
parsed_parent_dir = safe_urlparse(parent_dir)
378+
join_type = JoinType.from_parsed_uri(parsed_parent_dir)
379+
377380
if is_root or self.catalog_template is None:
378381
return self.fallback_strategy.get_catalog_href(cat, parent_dir, is_root)
379382
else:
380383
template_path = self.catalog_template.substitute(cat)
381384
if not template_path.endswith(".json"):
382-
template_path = os.path.join(template_path, cat.DEFAULT_FILE_NAME)
385+
template_path = join_path_or_url(
386+
join_type, template_path, cat.DEFAULT_FILE_NAME
387+
)
383388

384-
return os.path.join(parent_dir, template_path)
389+
return join_path_or_url(join_type, parent_dir, template_path)
385390

386391
def get_collection_href(
387392
self, col: "Collection_Type", parent_dir: str, is_root: bool
388393
) -> str:
394+
parsed_parent_dir = safe_urlparse(parent_dir)
395+
join_type = JoinType.from_parsed_uri(parsed_parent_dir)
396+
389397
if is_root or self.collection_template is None:
390398
return self.fallback_strategy.get_collection_href(col, parent_dir, is_root)
391399
else:
392400
template_path = self.collection_template.substitute(col)
393401
if not template_path.endswith(".json"):
394-
template_path = os.path.join(template_path, col.DEFAULT_FILE_NAME)
402+
template_path = join_path_or_url(
403+
join_type, template_path, col.DEFAULT_FILE_NAME
404+
)
395405

396-
return os.path.join(parent_dir, template_path)
406+
return join_path_or_url(join_type, parent_dir, template_path)
397407

398408
def get_item_href(self, item: "Item_Type", parent_dir: str) -> str:
409+
parsed_parent_dir = safe_urlparse(parent_dir)
410+
join_type = JoinType.from_parsed_uri(parsed_parent_dir)
411+
399412
if self.item_template is None:
400413
return self.fallback_strategy.get_item_href(item, parent_dir)
401414
else:
402415
template_path = self.item_template.substitute(item)
403416
if not template_path.endswith(".json"):
404-
template_path = os.path.join(template_path, "{}.json".format(item.id))
417+
template_path = join_path_or_url(
418+
join_type, template_path, "{}.json".format(item.id)
419+
)
405420

406-
return os.path.join(parent_dir, template_path)
421+
return join_path_or_url(join_type, parent_dir, template_path)
407422

408423

409424
class BestPracticesLayoutStrategy(HrefLayoutStrategy):
@@ -424,24 +439,33 @@ class BestPracticesLayoutStrategy(HrefLayoutStrategy):
424439
def get_catalog_href(
425440
self, cat: "Catalog_Type", parent_dir: str, is_root: bool
426441
) -> str:
442+
parsed_parent_dir = safe_urlparse(parent_dir)
443+
join_type = JoinType.from_parsed_uri(parsed_parent_dir)
444+
427445
if is_root:
428446
cat_root = parent_dir
429447
else:
430-
cat_root = os.path.join(parent_dir, "{}".format(cat.id))
448+
cat_root = join_path_or_url(join_type, parent_dir, "{}".format(cat.id))
431449

432-
return os.path.join(cat_root, cat.DEFAULT_FILE_NAME)
450+
return join_path_or_url(join_type, cat_root, cat.DEFAULT_FILE_NAME)
433451

434452
def get_collection_href(
435453
self, col: "Collection_Type", parent_dir: str, is_root: bool
436454
) -> str:
455+
parsed_parent_dir = safe_urlparse(parent_dir)
456+
join_type = JoinType.from_parsed_uri(parsed_parent_dir)
457+
437458
if is_root:
438459
col_root = parent_dir
439460
else:
440-
col_root = os.path.join(parent_dir, "{}".format(col.id))
461+
col_root = join_path_or_url(join_type, parent_dir, "{}".format(col.id))
441462

442-
return os.path.join(col_root, col.DEFAULT_FILE_NAME)
463+
return join_path_or_url(join_type, col_root, col.DEFAULT_FILE_NAME)
443464

444465
def get_item_href(self, item: "Item_Type", parent_dir: str) -> str:
445-
item_root = os.path.join(parent_dir, "{}".format(item.id))
466+
parsed_parent_dir = safe_urlparse(parent_dir)
467+
join_type = JoinType.from_parsed_uri(parsed_parent_dir)
468+
469+
item_root = join_path_or_url(join_type, parent_dir, "{}".format(item.id))
446470

447-
return os.path.join(item_root, "{}.json".format(item.id))
471+
return join_path_or_url(join_type, item_root, "{}.json".format(item.id))

‎pystac/stac_io.py

Lines changed: 4 additions & 5 deletions
Original file line numberDiff line numberDiff line change
@@ -14,11 +14,11 @@
1414
)
1515
import warnings
1616

17-
from urllib.parse import urlparse
1817
from urllib.request import urlopen
1918
from urllib.error import HTTPError
2019

2120
import pystac
21+
from pystac.utils import safe_urlparse
2222
import pystac.serialization
2323

2424
# Use orjson if available
@@ -182,7 +182,7 @@ def read_text(
182182
return self.read_text_from_href(href, *args, **kwargs)
183183

184184
def read_text_from_href(self, href: str, *args: Any, **kwargs: Any) -> str:
185-
parsed = urlparse(href)
185+
parsed = safe_urlparse(href)
186186
href_contents: str
187187
if parsed.scheme != "":
188188
try:
@@ -191,9 +191,8 @@ def read_text_from_href(self, href: str, *args: Any, **kwargs: Any) -> str:
191191
except HTTPError as e:
192192
raise Exception("Could not read uri {}".format(href)) from e
193193
else:
194-
with open(href) as f:
194+
with open(href, encoding="utf-8") as f:
195195
href_contents = f.read()
196-
197196
return href_contents
198197

199198
def write_text(
@@ -214,7 +213,7 @@ def write_text_to_href(
214213
dirname = os.path.dirname(href)
215214
if dirname != "" and not os.path.isdir(dirname):
216215
os.makedirs(dirname)
217-
with open(href, "w") as f:
216+
with open(href, "w", encoding="utf-8") as f:
218217
f.write(txt)
219218

220219

0 commit comments

Comments
 (0)