Skip to content

Commit 49dc73a

Browse files
valeriabarradennisYatuninsimonbyrnesriharshakandalacharleskawczynski
committed
Some WIP from the Software meeting notes
Co-authored-by: Dennis Yatunin <dyatun@gmail.com> Co-authored-by: Simon Byrne <simonbyrne@gmail.com> Co-authored-by: Sriharsha Kandala <sriharsha.kvs@gmail.com> Co-authored-by: Charles Kawczynski <kawczynski.charles@gmail.com>
1 parent 35f02b0 commit 49dc73a

File tree

2 files changed

+304
-16
lines changed

2 files changed

+304
-16
lines changed

‎docs/papers/joss/paper.bib

Lines changed: 160 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -0,0 +1,160 @@
1+
2+
@article{julia-paper,
3+
author = {Bezanson, Jeff and Edelman, Alan and Karpinski, Stefan and Shah, Viral B.},
4+
title = {Julia: A Fresh Approach to Numerical Computing},
5+
journal = {SIAM Review},
6+
volume = {59},
7+
number = {1},
8+
pages = {65-98},
9+
year = {2017},
10+
doi = {10.1137/141000671}
11+
}
12+
13+
14+
@misc{E3SM,
15+
title = {Energy Exascale Earth System Model v2.0},
16+
author = {{E3SM} Project, DOE},
17+
doi = {10.11578/E3SM/dc.20210927.1},
18+
url = {https://doi.org/10.11578/E3SM/dc.20210927.1},
19+
howpublished = {[Computer Software] \url{https://doi.org/10.11578/E3SM/dc.20210927.1}},
20+
year = {2021}
21+
}
22+
23+
@Misc{petsc-web-page,
24+
Author = "Satish Balay and Shrirang Abhyankar and Mark~F. Adams and Jed Brown
25+
and Peter Brune and Kris Buschelman and Lisandro Dalcin and Alp Dener and Victor Eijkhout
26+
and William~D. Gropp and Dinesh Kaushik and Matthew~G. Knepley and Dave~A. May
27+
and Lois Curfman McInnes and Richard Tran Mills and Todd Munson and Karl Rupp
28+
and Patrick Sanan and Barry~F. Smith and Stefano Zampini and Hong Zhang and Hong Zhang",
29+
Title = "{PETS}c {W}eb page",
30+
Note = "https://www.mcs.anl.gov/petsc",
31+
Year = "2020"}
32+
33+
@TechReport{petsc-user-ref,
34+
Author = "Satish Balay and Shrirang Abhyankar and Mark~F. Adams and Jed Brown
35+
and Peter Brune and Kris Buschelman and Lisandro Dalcin and Alp Dener and Victor Eijkhout
36+
and William~D. Gropp and Dinesh Kaushik and Matthew~G. Knepley and Dave~A. May
37+
and Lois Curfman McInnes and Richard Tran Mills and Todd Munson and Karl Rupp
38+
and Patrick Sanan and Barry~F. Smith and Stefano Zampini and Hong Zhang and Hong Zhang",
39+
Title = "{PETS}c Users Manual",
40+
Number = "ANL-95/11 - Revision 3.13",
41+
Institution = "Argonne National Laboratory",
42+
Year = "2020"}
43+
44+
@InProceedings{petsc-efficient,
45+
Author = "Satish Balay and William D. Gropp and Lois C. McInnes and Barry F. Smith",
46+
Title = "Efficient Management of Parallelism in Object Oriented
47+
Numerical Software Libraries",
48+
Booktitle = "Modern Software Tools in Scientific Computing",
49+
Editor = "E. Arge and A. M. Bruaset and H. P. Langtangen",
50+
Pages = "163--202",
51+
Publisher = "Birkhauser Press",
52+
Year = "1997"}
53+
54+
@article{libceed-joss-paper,
55+
author = {Jed Brown and Ahmad Abdelfattah and Valeria Barra and Natalie Beams and Jean Sylvain Camier and Veselin Dobrev and Yohann Dudouit and Leila Ghaffari and Tzanio Kolev and David Medina and Will Pazner and Thilina Ratnayaka and Jeremy Thompson and Stan Tomov},
56+
title = {{libCEED}: Fast algebra for high-order element-based discretizations},
57+
journal = {Journal of Open Source Software},
58+
year = {2021},
59+
publisher = {The Open Journal},
60+
volume = {6},
61+
number = {63},
62+
pages = {2945},
63+
doi = {10.21105/joss.02945}
64+
}
65+
66+
@misc{libceed-user-manual,
67+
author = {Abdelfattah, Ahmad and
68+
Barra, Valeria and
69+
Beams, Natalie and
70+
Brown, Jed and
71+
Camier, Jean-Sylvain and
72+
Dobrev, Veselin and
73+
Dudouit, Yohann and
74+
Ghaffari, Leila and
75+
Kolev, Tzanio and
76+
Medina, David and
77+
Pazner, Will and
78+
Ratnayaka, Thilina and
79+
Shakeri, Rezgar and
80+
Thompson, Jeremy L and
81+
Tomov, Stanimire and
82+
Wright III, James},
83+
title = {{libCEED} User Manual},
84+
month = dec,
85+
year = 2022,
86+
publisher = {Zenodo},
87+
version = {0.11.0},
88+
doi = {10.5281/zenodo.7480454}
89+
}
90+
91+
@misc{MFEMlibrary,
92+
title = "{MFEM}: {Modular Finite Element Methods Library}",
93+
url = "https://mfem.org/",
94+
year = "2021",
95+
doi = "10.11578/dc.20171025.1248"
96+
}
97+
98+
@article{mfem-paper,
99+
title = {{MFEM}: A Modular Finite Element Library},
100+
author = {R. Anderson and J. Andrej and A. Barker and J. Bramwell and J.-S. Camier and
101+
J. Cerveny V. Dobrev and Y. Dudouit and A. Fisher and Tz. Kolev and W. Pazner and
102+
M. Stowell and V. Tomov and I. Akkerman and J. Dahm and D. Medina and S. Zampini},
103+
journal = {Computers \& Mathematics with Applications},
104+
doi = {10.1016/j.camwa.2020.06.009},
105+
year = {2020}
106+
}
107+
108+
@article{dealII92,
109+
title = {The \texttt{deal.II} Library, Version 9.2},
110+
author = {Daniel Arndt and Wolfgang Bangerth and Bruno Blais and
111+
Thomas C. Clevenger and Marc Fehling and Alexander V. Grayver and
112+
Timo Heister and Luca Heltai and Martin Kronbichler and Matthias Maier and
113+
Peter Munch and Jean-Paul Pelteret and Reza Rastak and
114+
Ignacio Thomas and Bruno Turcksin and Zhuoran Wang and David Wells},
115+
journal = {Journal of Numerical Mathematics},
116+
publisher = {De Gruyter},
117+
year = {2020},
118+
volume = {28},
119+
number = {3},
120+
pages = {131-146},
121+
DOI = {10.1515/jnma-2020-0043},
122+
url = {https://dealii.org/deal92-preprint.pdf}
123+
}
124+
125+
@article{firedrake,
126+
author = {Rathgeber, Florian and Ham, David A. and Mitchell, Lawrence and Lange, Michael and Luporini, Fabio and Mcrae, Andrew T. T. and Bercea, Gheorghe-Teodor and Markall, Graham R. and Kelly, Paul H. J.},
127+
title = {Firedrake: Automating the Finite Element Method by Composing Abstractions},
128+
year = {2016},
129+
issue_date = {September 2017},
130+
publisher = {Association for Computing Machinery},
131+
address = {New York, NY, USA},
132+
volume = {43},
133+
number = {3},
134+
issn = {0098-3500},
135+
url = {https://doi.org/10.1145/2998441},
136+
doi = {10.1145/2998441},
137+
journal = {ACM Trans. Math. Softw.}
138+
}
139+
140+
@article{FeniCS,
141+
author = {Alnæs, Martin S. and Blechta, Jan and Hake, Johan and Johansson, August and Kehlet, Benjamin and Logg, Anders and Richardson, Chris and Ring, Johannes and Rognes, Marie E. and Wells, Garth N.},
142+
title = {The FEniCS Project Version 1.5},
143+
year = {2015},
144+
volume = {3},
145+
number = {100},
146+
pages = {9-23},
147+
doi = {https://doi.org/10.11588/ans.2015.100.20553},
148+
journal = {Archive of Numerical Software}
149+
}
150+
151+
@software{climate_machine_zenodo,
152+
author = {Climate Modeling Alliance},
153+
title = {ClimateMachine.jl},
154+
month = may,
155+
year = 2020,
156+
publisher = {Zenodo},
157+
version = {0.2.0},
158+
doi = {10.5281/zenodo.5542395},
159+
url = {https://doi.org/10.5281/zenodo.5542395}
160+
}

‎docs/papers/joss/paper.md

Lines changed: 144 additions & 16 deletions
Original file line numberDiff line numberDiff line change
@@ -8,48 +8,176 @@ tags:
88
- staggered finite differences
99
- matrix-free
1010
authors:
11-
- name: Foo
11+
- name: Simon Byrne
1212
orcid: XXXX
1313
affiliation: 1
14-
- name: Bar
15-
orcid: YYYY
14+
- name: Charles Kawczynski
15+
orcid: XXXX
16+
affiliation: 1
17+
- name: Valeria Barra
18+
orcid: 0000-0003-1129-2056
19+
affiliation: 1
20+
- name: Jake Bolewski
21+
orcid: XXXX
22+
affiliation: 1
23+
- name: Sriharsha Kandala
24+
orcid: XXXX
25+
affiliation: 1
26+
- name: Zhaoyi Shen
27+
orcid: XXXX
28+
affiliation: 1
29+
- name: Jia He
30+
orcid: XXXX
31+
affiliation: 1
32+
- name: Dennis Yatunin
33+
orcid: XXXX
34+
affiliation: 1
35+
- name: Ben Mackay
36+
orcid: XXXX
37+
affiliation: 1
38+
- name: Akshay Sridhar
39+
orcid: XXXX
40+
affiliation: 1
41+
- name: Simon Danish ?
42+
orcid: XXXX
1643
affiliation: 2
44+
- name: Kiran Pamnany
45+
orcid: XXXX
46+
affiliation: 3
47+
- name: Toby Bischoff
48+
orcid: XXXX
49+
affiliation: 1
50+
- name: LenkaNovak
51+
orcid: XXXX
52+
affiliation: 1
53+
- name: Julia Sloan
54+
orcid: XXXX
55+
affiliation: 1
56+
- name: Daniel (Zhengyu) Huang
57+
orcid: XXXX
58+
affiliation: 1
59+
- name: Oswald Knoth
60+
orcid: XXXX
61+
affiliation: 4
62+
- name: Paul Ullrich
63+
orcid: XXXX
64+
affiliation: 5
65+
-name: Tapio Schneider
66+
orcid: XXXX
67+
affiliation: 1
68+
1769
affiliations:
1870
- name: California Institute of Technology
1971
index: 1
20-
- name: Other Institution
72+
- name: ?
2173
index: 2
74+
- name: Relational AI
75+
index: 3
76+
- name: TROPOS
77+
index: 4
78+
- name: University of California Davis
79+
index: 5
2280
date: 10 January 2023
2381
bibliography: paper.bib
2482
---
2583

26-
# Motivation and Benefits
27-
A "white paper" that briefly mentions all the nice properties of the library: extensibility, composability, ease-of-use, library-reuse, (performance-portability?), (scalability?). This will give ClimaCore its proper DOI and citability tool.
84+
<!-- The list of authors of the software and their affiliations, using the correct format was generated from https://github.com/CliMA/ClimaCore.jl/graphs/contributors + Paul, Oswald and Tapio. The order of co-authors reflects the order of GitHub contributions by commits. Contributors with only one commit are not included at the moment. We can revise this policy if anyone does not agree with it. Also, anyone included who does not wish to be included can be removed, of course. -->
85+
86+
# Summary
87+
88+
<!-- A summary describing the high-level functionality and purpose of the software for a diverse, non-specialist audience. -->
89+
90+
The Climate Modelling Alliance ([CliMA](https://clima.caltech.edu/)) is developing a new Earth System Model (ESM), entirely written in the [Julia](https://julialang.org/) programming language [@julia-paper]. The main goal of the project is to build an ESM that automatically learns from diverse data sources to produce accurate climate predictions with quantified uncertainties.
91+
92+
`ClimaCore.jl` is a new open-source software library that provides a suite of tools for constructing spatial discretizations, underlying many of the CliMA model components and providing the dynamical core (_dycore_) for the atmosphere and land components of the ESM. It is designed with a high-level application programming interface (API), which facilitates modularity, composition of differential operators, definition of flexible discretizations, and library reuse.
93+
94+
# Statement of need
95+
96+
<!-- A Statement of need section that clearly illustrates the research purpose of the software and places it in the context of related work. -->
97+
98+
Earth system model dynamical cores are traditionally hard-coded to specific equation sets, with fixed spatial and temporal discretizations, and specific geometries, such as spherical geometries for general circulation models (GCM) or Cartesian ones for large-eddy simulations (LES) (see, for instance, the High Order Method Modeling Environment (HOMME) used by the Energy Exascale Earth System Model (E3SM) [@E3SM]).
99+
100+
`ClimaCore.jl` aims to be a more flexible approach, inspired by other mathematical software libraries for constructing spatial discretizations of partial differential equations (PDEs), such as PETSc [@petsc-web-page; @petsc-user-ref; @petsc-efficient], libCEED [@libceed-joss-paper; @libceed-user-manual], MFEM [@MFEMlibrary; @mfem-paper], deal.II [@dealII92], Firedrake [@firedrake], and FeniCS [@FeniCS].
101+
102+
However, ESMs tend to have some specific properties, some of which can leverage modern heterogenous architectures (including CPUs and GPUs) or modern ML/AI tools, that there are advantages to developing a new library
103+
104+
- very skewed aspect ratio for the atmosphere component: O(100km) in the horizontal vs O(10m) in the vertical;
105+
- implicit-explicit (IMEX) timestepping, with only the vertical parts handled implicitly: horizontally-explicit, vertically-implicit (HEVI) schemes;
106+
- use of different discertizations in each dimension, for example our current atmosphere model uses a specral element discretization in the horizontal, with a staggered finite difference discretization in the verfical;
107+
- don't need a fully unstructured mesh: 3D meshes are constucted by extruding a 2D mesh;
108+
- distributed parallely only in the horizontal direction;
109+
- support both Cartesian and spherical geometries and vector bases: on a sphere, vector components are typical specified in spherical basis (zonal, meridonal, radial);
110+
- capability to run embedded regional high-resolution and global simulations.
111+
112+
113+
114+
<!-- A list of key references, including to other software addressing related needs. Note that the references should include full names of venues, e.g., journals and conferences, not abbreviations only understood in the context of a specific discipline. -->
115+
116+
<!-- Mention (if applicable) a representative set of past or ongoing research projects using the software and recent scholarly publications enabled by it. -->
117+
118+
`ClimaCore.jl` is currently being used as the basis for the atmosphere and land model components of the CliMA earth system model.
119+
120+
121+
122+
# Introduction
123+
124+
125+
126+
<!-- from README -->
127+
`ClimaCore.jl` is a the dynamical core (_dycore_) of the atmosphere and land models, providing discretization tools to solve the governing equations of the ESM component models.
128+
129+
`ClimaCore.jl`'s high-level API facilitates modularity and composition of differential operators and the definition of flexible discretizations. This, in turn, is coupled with low-level APIs that support different data layouts, specialized implementations, and flexible models for threading, to better face high-performance optimization, data storage, and scalability challenges on modern HPC architectures. `ClimaCore.jl` is designed to be performance portable and can be used in a distributed setting with CPU and GPU clusters.
130+
131+
## Technical aims and current support
132+
* Support both large-eddy simulation (LES) and general circulation model (GCM) configurations for the atmosphere.
133+
* A suite of tools for constructing space discretizations.
134+
* Horizontal spectral elements:
135+
- Supports both continuous Galerkin (CG) and discontinuous Galerkin (DG) spectral element discretizations.
136+
* Flexible choice of vertical discretization (currently staggered finite differences)
137+
* Support for different geometries (Cartesian, spherical), with governing equations discretized in terms of covariant vectors for curvilinear, non-orthogonal systems and Cartesian vectors for Euclidean spaces.
138+
* `Field` abstraction:
139+
- Scalar, vector or struct-valued
140+
- Stores values, geometry, and mesh information
141+
- Flexible memory layouts: Array-of-Structs (AoS), Struct-of-Arrays (SoA),Array-of-Struct-of-Arrays (AoSoA)
142+
- Useful overloads: `sum` (integral), `norm`, etc.
143+
- Compatible with [`DifferentialEquations.jl`](https://diffeq.sciml.ai/stable/) time steppers.
144+
* Composable operators via broadcasting: apply a function element-wise to an array; scalar values are broadcast over arrays
145+
* Fusion of multiple operations; can be specialized for custom functions or argument types (e.g. `CuArray` compiles and applies a custom CUDA kernel).
146+
* Operators (`grad`, `div`, `interpolate`) are “pseudo-functions”: Act like functions when broadcasted over a `Field`; fuse operators and function calls.
147+
* Add element node size dimensions to type domain
148+
- i.e., specialize on polynomial degree
149+
- important for GPU kernel performance.
150+
* Flexible memory layouts allow for flexible threading models (upcoming):
151+
- CPU thread over elements
152+
- GPU thread over nodes.
153+
154+
28155

29-
# Costs
30-
From JOSS' [website](https://joss.readthedocs.io/en/latest/submitting.html#submitting-a-paper-to-joss): "If you’ve already developed a fully featured research code, released it under an OSI-approved license, and written good documentation and tests, then we expect that it should take perhaps an hour or two to prepare and submit your paper to JOSS. But please read these instructions carefully for a streamlined submission."
31156

32157
# Tentative TODOs
158+
Draft of a "white paper" that briefly mentions all the nice properties of the library: extensibility, composability, ease-of-use, library-reuse, (performance-portability?), (scalability?). This will give ClimaCore its proper DOI and citability tool.
159+
33160
Improve Docs:
34161
- [x] Getting started/How to guide
35-
- [x] Contributing guide + Authorship policy + Code of Conduct (?)
162+
- [x] Contributing guide + Code of Conduct (?)
36163
- [ ] Examples documentation (equations set, what to expect from each example, artifacts, if included)
37164

38165

39166
Improve Unit Tests:
40167
- [ ] Unit tests: strive for best code coverage: e.g., double check that all operators are tested
41168

169+
Examples:
170+
- [ ] Address memory usage and OoM issues when examples are run locally
42171

43-
Performance:
44-
- [ ] Distributed computing capability (probably no need for scaling studies)
45-
- [ ] Multi-threading capability (might include initial GPU support?)
46172

47-
# Summary and statement of need
48-
49-
Lorem Ipsum
173+
Performance:
174+
- [ ] Distributed computing capability (re-run latest scaling studies)
175+
- [ ] Multi-threading capability?
176+
- [ ] Might include initial GPU support?
50177

51178

179+
<!-- Acknowledgement of any financial support. -->
52180
# Acknowledgements
53-
181+
We acknowledge contributions from several others who played a role in the evolution of this library, especially contributors and users of an eralier iteration of this effort, [ClimateMachine.jl](https://github.com/CliMA/ClimateMachine.jl) [@climate_machine_zenodo]. The development of this package was supported by the generosity of Eric and Wendy Schmidt by recommendation of the Schmidt Futures program, and by the Defense Advanced Research Projects Agency (Agreement No. HR00112290030).
54182

55183
# References

0 commit comments

Comments
 (0)