Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
The table of contents is too big for display.
Diff view
Diff view
  •  
  •  
  •  
33 changes: 1 addition & 32 deletions mindnlp/__init__.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,44 +17,13 @@
MindNLP library.
"""
import os
import platform

# huggingface env
if os.environ.get('HF_ENDPOINT', None) is None:
os.environ["HF_ENDPOINT"] = 'https://hf-mirror.com'

# for huawei cloud modelarts
if 'RANK_TABLE_FILE' in os.environ:
del os.environ['RANK_TABLE_FILE']

import mindspore
from mindspore._c_expression import MSContext # pylint: disable=no-name-in-module, import-error
try:
from mindspore._c_expression import disable_multi_thread
except:
disable_multi_thread = None

if os.environ.get('DEVICE_TARGET', None) is not None:
mindspore.set_device(os.environ.get('DEVICE_TARGET'))

# for different ascend devices
if platform.system().lower() == 'linux' and mindspore.get_context('device_target') == 'Ascend':
SOC = MSContext.get_instance().get_ascend_soc_version()
# enable vmm since only vmm can release device memory when del tensor.
if SOC != 'ascend310b':
os.environ["MS_ALLOC_CONF"] = 'enable_vmm:True,vmm_align_size:2MB'

if SOC in ('ascend910', 'ascend310b'):
# context.set_context(ascend_config={"precision_mode": "allow_mix_precision"})
mindspore.device_context.ascend.op_precision.precision_mode('allow_mix_precision')
if SOC == 'ascend310b' and disable_multi_thread is not None:
disable_multi_thread()

# set mindnlp.core to torch
from .utils.torch_proxy import initialize_torch_proxy, setup_metadata_patch
initialize_torch_proxy()
setup_metadata_patch()

import mindtorch
from .utils.safetensors_patch import setup_safetensors_patch
setup_safetensors_patch()

Expand Down
79 changes: 0 additions & 79 deletions mindnlp/core/cuda/amp/autocast_mode.py

This file was deleted.

19 changes: 0 additions & 19 deletions mindnlp/core/distributed/_shard/checkpoint/__init__.py

This file was deleted.

13 changes: 0 additions & 13 deletions mindnlp/core/distributed/_shard/sharded_tensor/_ops/__init__.py

This file was deleted.

21 changes: 0 additions & 21 deletions mindnlp/core/distributed/_sharded_tensor/__init__.py

This file was deleted.

22 changes: 0 additions & 22 deletions mindnlp/core/distributed/_sharding_spec/__init__.py

This file was deleted.

9 changes: 0 additions & 9 deletions mindnlp/core/distributed/_tensor/api.py

This file was deleted.

10 changes: 0 additions & 10 deletions mindnlp/core/distributed/_tensor/placement_types.py

This file was deleted.

7 changes: 0 additions & 7 deletions mindnlp/core/distributed/pipelining/README.md

This file was deleted.

20 changes: 0 additions & 20 deletions mindnlp/core/distributed/rpc/_testing/__init__.py

This file was deleted.

90 changes: 0 additions & 90 deletions mindnlp/core/npu/amp/autocast_mode.py

This file was deleted.

10 changes: 0 additions & 10 deletions mindnlp/core/utils/data/__init__.py

This file was deleted.

Loading
Loading