Skip to content

feat: make FileType(s) public #158

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
merged 5 commits into from
Dec 3, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
53 changes: 17 additions & 36 deletions nominal/_utils.py
Original file line number Diff line number Diff line change
@@ -1,54 +1,35 @@
from __future__ import annotations

import logging
import mimetypes
import os
from contextlib import contextmanager
from pathlib import Path
from typing import BinaryIO, Callable, Iterator, NamedTuple, TypeVar
from typing import Any, BinaryIO, Callable, Iterator, TypeVar

from typing_extensions import ParamSpec

from nominal.core import filetype

logger = logging.getLogger(__name__)


Param = ParamSpec("Param")
T = TypeVar("T")


class FileType(NamedTuple):
extension: str
mimetype: str

@classmethod
def from_path(cls, path: Path | str, default_mimetype: str = "application/octect-stream") -> FileType:
ext = "".join(Path(path).suffixes)
mimetype, _encoding = mimetypes.guess_type(path)
if mimetype is None:
return cls(ext, default_mimetype)
return cls(ext, mimetype)

@classmethod
def from_path_dataset(cls, path: Path | str) -> FileType:
path_string = str(path) if isinstance(path, Path) else path
if path_string.endswith(".csv"):
return FileTypes.CSV
if path_string.endswith(".csv.gz"):
return FileTypes.CSV_GZ
if path_string.endswith(".parquet"):
return FileTypes.PARQUET
raise ValueError(f"dataset path '{path}' must end in .csv, .csv.gz, or .parquet")


class FileTypes:
BINARY: FileType = FileType("", "application/octet-stream")
CSV: FileType = FileType(".csv", "text/csv")
CSV_GZ: FileType = FileType(".csv.gz", "text/csv")
JSON: FileType = FileType(".json", "application/json")
MP4: FileType = FileType(".mp4", "video/mp4")
MCAP: FileType = FileType(".mcap", "application/octet-stream")
# https://issues.apache.org/jira/browse/PARQUET-1889
PARQUET: FileType = FileType(".parquet", "application/vnd.apache.parquet")
def __getattr__(attr: str) -> Any:
import warnings

deprecated_attrs = {"FileType": filetype.FileType, "FileTypes": filetype.FileTypes}
if attr in deprecated_attrs:
warnings.warn(
(
f"nominal._utils.{attr} is deprecated and will be removed in a future version, use "
f"nominal.core.{attr} instead."
),
UserWarning,
stacklevel=2,
)
return deprecated_attrs[attr]


@contextmanager
Expand Down
2 changes: 1 addition & 1 deletion nominal/cli/attachment.py
Original file line number Diff line number Diff line change
Expand Up @@ -4,9 +4,9 @@

import click

from nominal._utils import FileType
from nominal.cli.util.global_decorators import client_options, global_options
from nominal.core.client import NominalClient
from nominal.core.filetype import FileType


@click.group(name="attachment")
Expand Down
3 changes: 3 additions & 0 deletions nominal/core/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
from nominal.core.client import NominalClient
from nominal.core.connection import Connection
from nominal.core.dataset import Dataset, poll_until_ingestion_completed
from nominal.core.filetype import FileType, FileTypes
from nominal.core.log import Log, LogSet
from nominal.core.run import Run
from nominal.core.user import User
Expand All @@ -20,6 +21,8 @@
"ChecklistBuilder",
"Connection",
"Dataset",
"FileType",
"FileTypes",
"Log",
"LogSet",
"NominalClient",
Expand Down
2 changes: 1 addition & 1 deletion nominal/core/_multipart.py
Original file line number Diff line number Diff line change
Expand Up @@ -11,7 +11,7 @@
import requests

from nominal._api.scout_service_api import ingest_api, upload_api
from nominal._utils import FileType
from nominal.core.filetype import FileType
from nominal.exceptions import NominalMultipartUploadFailed

logger = logging.getLogger(__name__)
Expand Down
10 changes: 3 additions & 7 deletions nominal/core/client.py
Original file line number Diff line number Diff line change
Expand Up @@ -28,20 +28,18 @@
storage_datasource_api,
timeseries_logicalseries_api,
)
from nominal._utils import (
FileType,
FileTypes,
deprecate_keyword_argument,
)
from nominal._utils import deprecate_keyword_argument
from nominal.core._clientsbunch import ClientsBunch
from nominal.core._conjure_utils import _available_units, _build_unit_update
from nominal.core._multipart import upload_multipart_file, upload_multipart_io
from nominal.core._utils import construct_user_agent_string, rid_from_instance_or_string
from nominal.core.asset import Asset
from nominal.core.attachment import Attachment, _iter_get_attachments
from nominal.core.channel import Channel
from nominal.core.checklist import Checklist, ChecklistBuilder
from nominal.core.connection import Connection
from nominal.core.dataset import Dataset, _get_dataset, _get_datasets
from nominal.core.filetype import FileType, FileTypes
from nominal.core.log import Log, LogSet, _get_log_set
from nominal.core.run import Run
from nominal.core.unit import Unit
Expand All @@ -57,8 +55,6 @@
_to_typed_timestamp_type,
)

from .asset import Asset

logger = logging.getLogger(__name__)


Expand Down
2 changes: 1 addition & 1 deletion nominal/core/dataset.py
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,12 @@
timeseries_logicalseries_api,
upload_api,
)
from nominal._utils import FileType, FileTypes
from nominal.core._clientsbunch import HasAuthHeader
from nominal.core._conjure_utils import _available_units, _build_unit_update
from nominal.core._multipart import upload_multipart_io
from nominal.core._utils import HasRid, update_dataclass
from nominal.core.channel import Channel, _get_series_values_csv
from nominal.core.filetype import FileType, FileTypes
from nominal.exceptions import NominalIngestError, NominalIngestFailed, NominalIngestMultiError
from nominal.ts import (
_MAX_TIMESTAMP,
Expand Down
43 changes: 43 additions & 0 deletions nominal/core/filetype.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,43 @@
from __future__ import annotations

import logging
import mimetypes
from pathlib import Path
from typing import NamedTuple

logger = logging.getLogger(__name__)


class FileType(NamedTuple):
extension: str
mimetype: str

@classmethod
def from_path(cls, path: Path | str, default_mimetype: str = "application/octect-stream") -> FileType:
ext = "".join(Path(path).suffixes)
mimetype, _encoding = mimetypes.guess_type(path)
if mimetype is None:
return cls(ext, default_mimetype)
return cls(ext, mimetype)

@classmethod
def from_path_dataset(cls, path: Path | str) -> FileType:
path_string = str(path) if isinstance(path, Path) else path
if path_string.endswith(".csv"):
return FileTypes.CSV
if path_string.endswith(".csv.gz"):
return FileTypes.CSV_GZ
if path_string.endswith(".parquet"):
return FileTypes.PARQUET
raise ValueError(f"dataset path '{path}' must end in .csv, .csv.gz, or .parquet")


class FileTypes:
BINARY: FileType = FileType("", "application/octet-stream")
CSV: FileType = FileType(".csv", "text/csv")
CSV_GZ: FileType = FileType(".csv.gz", "text/csv")
JSON: FileType = FileType(".json", "application/json")
MP4: FileType = FileType(".mp4", "video/mp4")
MCAP: FileType = FileType(".mcap", "application/octet-stream")
# https://issues.apache.org/jira/browse/PARQUET-1889
PARQUET: FileType = FileType(".parquet", "application/vnd.apache.parquet")
4 changes: 3 additions & 1 deletion nominal/nominal.py
Original file line number Diff line number Diff line change
Expand Up @@ -7,13 +7,15 @@
from typing import TYPE_CHECKING, BinaryIO, Iterable, Mapping, Sequence

from nominal import Connection, _config, ts
from nominal._utils import FileType, FileTypes, deprecate_keyword_argument, reader_writer
from nominal._utils import deprecate_keyword_argument, reader_writer
from nominal.core import (
Asset,
Attachment,
Checklist,
ChecklistBuilder,
Dataset,
FileType,
FileTypes,
Log,
LogSet,
NominalClient,
Expand Down
Loading