Skip to content

Commit 13700c8

Browse files
thewtexthewtex
committed
ENH: Migrate data to IPFS
This improves reliability, addressing #395, but also the contributor experience, maintainability, and sustainability.
1 parent 06cc158 commit 13700c8

File tree

1,134 files changed

+1828
-579
lines changed

Some content is hidden

Large Commits have some content hidden by default. Use the searchbox below for content that may be hidden.

1,134 files changed

+1828
-579
lines changed

‎.gitattributes

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -12,4 +12,4 @@
1212

1313
# ExternalData content links must have LF newlines
1414
*.md5 crlf=input
15-
*.sha512 crlf=input
15+
*.cid crlf=input

‎.github/labeler.yml

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -83,7 +83,7 @@ type:Python notebook:
8383

8484
type:Data:
8585
files:
86-
- ".*.sha512"
86+
- ".*.cid"
8787

8888
type:Infrastructure:
8989
files:

‎CMake/ExternalData.cmake

Lines changed: 1202 additions & 0 deletions
Large diffs are not rendered by default.

‎CMake/ExternalData_config.cmake.in

Lines changed: 6 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,6 @@
1+
set(ExternalData_OBJECT_STORES "@ExternalData_OBJECT_STORES@")
2+
set(ExternalData_URL_TEMPLATES "@ExternalData_URL_TEMPLATES@")
3+
set(ExternalData_TIMEOUT_INACTIVITY "@ExternalData_TIMEOUT_INACTIVITY@")
4+
set(ExternalData_TIMEOUT_ABSOLUTE "@ExternalData_TIMEOUT_ABSOLUTE@")
5+
set(ExternalData_NO_SYMLINKS "@ExternalData_NO_SYMLINKS@")
6+
@_ExternalData_CONFIG_CODE@

‎CMake/ITKSphinxExamplesExternalData.cmake

Lines changed: 50 additions & 8 deletions
Original file line numberDiff line numberDiff line change
@@ -1,4 +1,5 @@
1-
include(ExternalData)
1+
get_filename_component(_ITKExternalData_DIR "${CMAKE_CURRENT_LIST_FILE}" PATH)
2+
include(${_ITKExternalData_DIR}/ExternalData.cmake)
23

34
if(NOT ExternalData_OBJECT_STORES)
45
# Use ExternalData_OBJECT_STORES from environment as default.
@@ -20,17 +21,58 @@ list(APPEND ExternalData_OBJECT_STORES
2021
"${CMAKE_SOURCE_DIR}/.ExternalData"
2122
)
2223

24+
set(ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID bafybeicjaaz3xi2qso3kdirmy5ixoxegvfp3bm7o44ytoh6egj5m6wgeuu)
25+
set(ExternalData_URL_ALGO_CID_lower cid)
2326
set(ExternalData_URL_TEMPLATES "" CACHE STRING
2427
"Additional URL templates for the ExternalData CMake script to look for testing data. E.g.
2528
file:///var/bigharddrive/%(algo)/%(hash)")
2629
mark_as_advanced(ExternalData_URL_TEMPLATES)
27-
list(APPEND ExternalData_URL_TEMPLATES
28-
# Data published on GitHub Pages
29-
"https://insightsoftwareconsortium.github.io/ITKTestingData/%(algo)/%(hash)"
30+
if(NOT ITK_FORBID_DOWNLOADS)
31+
list(APPEND ExternalData_URL_TEMPLATES
32+
# Local IPFS gateway
33+
"http://127.0.0.1:8080/ipfs/%(hash)"
3034

31-
# Data published on Girder
32-
"https://data.kitware.com:443/api/v1/file/hashsum/%(algo)/%(hash)/download"
33-
)
35+
# Released data rsync'd to Kitware's Apache web server
36+
"https://itk.org/files/ExternalData/%(algo)/%(hash)"
37+
38+
# Restricted gateway with released data
39+
"https://itk.mypinata.cloud/ipfs/${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}/Objects/CID/%(hash)"
40+
41+
# New data on web3.storage
42+
"https://w3s.link/ipfs/%(hash)"
43+
44+
# Released data on web3.storage
45+
"https://${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}.ipfs.w3s.link/Objects/CID/%(hash)"
46+
47+
# Released data on estuary.tech
48+
"https://${ITK_SPHINX_EXAMPLES_DATA_RELEASED_ROOT_CID}.ipfs.dweb.link/Objects/CID/%(hash)"
49+
50+
# Gateway for arbitrary new files, uploaded to web3.storage
51+
"https://dweb.link/ipfs/%(hash)"
52+
53+
# Protocol Labs gateway
54+
"https://ipfs.io/ipfs/%(hash)"
55+
56+
# Cloudflare gateway
57+
"https://cloudflare-ipfs.com/ipfs/%(hash)"
58+
59+
# DataLad repository on gin.g-node.org
60+
"https://gin.g-node.org/InsightSoftwareConsortium/ITKSphinxExamplesData/raw/main/Objects/CID/%(hash)"
61+
)
62+
endif()
3463

3564
# Tell ExternalData commands to transform raw files to content links.
36-
set(ExternalData_LINK_CONTENT SHA512)
65+
set(ExternalData_LINK_CONTENT CID)
66+
67+
# Emscripten currently has difficulty reading symlinks.
68+
if(EMSCRIPTEN)
69+
set(ExternalData_NO_SYMLINKS 1)
70+
endif()
71+
72+
# Match series of the form <base>.<ext>, <base>.<n>.<ext> such that <base> may
73+
# end in a (test) number that is not part of any series numbering.
74+
set(ExternalData_SERIES_PARSE "()(\\.[^./]*)$")
75+
set(ExternalData_SERIES_MATCH "(\\.[0-9]+)?")
76+
77+
# Sometimes we want to download very large files.
78+
set(ExternalData_TIMEOUT_ABSOLUTE 900)

‎CMake/ITKSphinxExamplesMacros.cmake

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -113,7 +113,7 @@ function(compare_to_baseline)
113113
endif()
114114

115115
file(GLOB baseline_image ${CMAKE_CURRENT_SOURCE_DIR}/${LOCAL_COMPARISON_EXAMPLE_NAME}/${LOCAL_COMPARISON_BASELINE_PREFIX}.*)
116-
string(REPLACE .sha512 "" baseline_image "${baseline_image}")
116+
string(REPLACE .cid "" baseline_image "${baseline_image}")
117117
get_filename_component(baseline_image_file "${baseline_image}" NAME)
118118
set(baseline_image "${CMAKE_CURRENT_BINARY_DIR}/${LOCAL_COMPARISON_EXAMPLE_NAME}/${baseline_image_file}")
119119

‎CMakeLists.txt

Lines changed: 2 additions & 2 deletions
Original file line numberDiff line numberDiff line change
@@ -125,12 +125,12 @@ set(content_links)
125125
# corresponding locations in the binary tree.
126126
foreach(_dir ${_RootSubDirs})
127127
file(GLOB_RECURSE ${_dir}_content_links
128-
RELATIVE "${ITKSphinxExamples_SOURCE_DIR}" "${_dir}/*.sha512")
128+
RELATIVE "${ITKSphinxExamples_SOURCE_DIR}" "${_dir}/*.cid")
129129
set(content_links ${content_links} ${${_dir}_content_links})
130130
endforeach()
131131

132132
foreach(link ${content_links})
133-
string(REGEX REPLACE "\\.sha512$" "" link ${link})
133+
string(REGEX REPLACE "\\.cid$" "" link ${link})
134134
ExternalData_Expand_Arguments(ITKSphinxExamplesData
135135
link_location
136136
DATA{${link}}

‎Documentation/Build/DownloadExampleHighlighted.png.cid

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bafkreihapw3rzwmabx6k4kdwqi4feovykoxceoqfhksxbowd6odj5qdvpu

‎Documentation/Build/DownloadExampleHighlighted.png.sha512

Lines changed: 0 additions & 1 deletion
This file was deleted.

‎Documentation/Build/ITK-SNAP.png.cid

Lines changed: 1 addition & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1 @@
1+
bafkreia54qdss3ndh2j6bnwxgwb3aye2a77ngle5z4p33tikmlcpjzh6qi

0 commit comments

Comments
 (0)