Skip to content
Open
Show file tree
Hide file tree
Changes from 3 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
52 changes: 51 additions & 1 deletion taskfile.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -16,6 +16,8 @@ vars:
G_CORE_COMPONENT_SUBMODULES_DIR: "{{.G_CORE_COMPONENT_DIR}}/submodules"
G_WEBUI_SRC_DIR: "{{.G_COMPONENTS_DIR}}/webui"

G_DATASET_LOCATION: "{{.ROOT_DIR}}/datasets/dataset.json"

# Build paths
G_BUILD_DIR: "{{.ROOT_DIR}}/build"
G_CORE_COMPONENT_BUILD_DIR: "{{.G_BUILD_DIR}}/core"
Expand Down Expand Up @@ -94,6 +96,7 @@ tasks:
vars:
CHECKSUM_FILE: "{{.G_BUILD_DIR}}/{{.TASK}}.md5"
OUTPUT_DIR: "{{.G_PACKAGE_BUILD_DIR}}"
BUILD_WITH_SYMBOLS: "{{.BUILD_WITH_SYMBOLS | default false}}"
sources:
- "{{.G_BUILD_DIR}}/package-venv.md5"
- "{{.G_BUILD_DIR}}/webui.md5"
Expand All @@ -111,7 +114,9 @@ tasks:
- "components/package-template/src/**/*"
generates: ["{{.CHECKSUM_FILE}}"]
deps:
- "core"
- task: "core"
vars:
BUILD_WITH_SYMBOLS: "{{.BUILD_WITH_SYMBOLS}}"
- "clp-package-utils"
- "clp-py-utils"
- "init"
Expand Down Expand Up @@ -204,11 +209,17 @@ tasks:
ref: ".OUTPUT_DIRS"

core:
vars:
BUILD_WITH_SYMBOLS: "{{.BUILD_WITH_SYMBOLS | default false}}"
cmds:
- task: "core-generate"
vars:
BUILD_WITH_SYMBOLS: "{{.BUILD_WITH_SYMBOLS}}"
- task: "core-build"

core-generate:
vars:
BUILD_WITH_SYMBOLS: "{{.BUILD_WITH_SYMBOLS | default false}}"
internal: true
sources: &core_source_files
- "{{.G_DEPS_CORE_CHECKSUM_FILE}}"
Expand All @@ -217,14 +228,23 @@ tasks:
- "{{.G_CORE_COMPONENT_DIR}}/src/**/*"
- "{{.TASKFILE}}"
- "/etc/os-release"
generates:
# ugly workaround so that when the var changes, the task gets re-run (triggering core-build
# re-run)
- ".task/build_with_symbols_{{.BUILD_WITH_SYMBOLS}}.stamp"
deps:
- "clp-s-generate-parsers"
- "deps:core"
cmds:
- task: "utils:cmake:generate"
vars:
EXTRA_ARGS:
- >-
{{if eq .BUILD_WITH_SYMBOLS "true"}}-DCMAKE_BUILD_TYPE=RelWithDebInfo{{end}}
BUILD_DIR: "{{.G_CORE_COMPONENT_BUILD_DIR}}"
SOURCE_DIR: "{{.G_CORE_COMPONENT_DIR}}"
- "rm .task/build_with_symbols_*.stamp"
- "touch .task/build_with_symbols_{{.BUILD_WITH_SYMBOLS}}.stamp"

core-build:
internal: true
Expand Down Expand Up @@ -586,6 +606,36 @@ tasks:
cmds:
- "rm -rf dist"

download-dataset:
status:
- "test -f {{.G_DATASET_LOCATION}}"
cmds:
- >-
curl -o output.tar.gz
https://zenodo.org/records/10516387/files/cockroachdb.tar.gz?download=1
- "tar -xvzf output.tar.gz"
- "rm output.tar.gz"
- >-
mkdir -p "$(dirname "{{.G_DATASET_LOCATION}}")"
- "mv cockroachdb/cockroach.node1.log {{.G_DATASET_LOCATION}}"
- "rmdir cockroachdb"

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

🛠️ Refactor suggestion

Download task lacks robustness & safety flags

  1. curl doesn’t follow redirects or fail on HTTP errors.
  2. No checksum is verified; corrupt or tampered tarballs will go unnoticed.
  3. Minor: using -v extraction is chatty and rm should tolerate a missing file.
-      - >-
-        curl -o output.tar.gz
-        https://zenodo.org/records/10516387/files/cockroachdb.tar.gz?download=1
-      - "tar -xvzf output.tar.gz"
-      - "rm output.tar.gz"
+      - >-
+        curl --fail --location --show-error --silent
+        --output output.tar.gz
+        https://zenodo.org/records/10516387/files/cockroachdb.tar.gz?download=1
+      # TODO: verify checksum here if reproducibility matters
+      - "tar -xzf output.tar.gz"
+      - "rm -f output.tar.gz"

Adding --fail --location --show-error --silent makes the download step safer and quieter; switching to -xzf avoids verbose logs; rm -f prevents an exit on missing file.

🤖 Prompt for AI Agents
In taskfile.yaml lines 609 to 622, improve the download-dataset task by adding
curl flags --fail, --location, --show-error, and --silent to handle redirects,
fail on HTTP errors, and reduce output noise. Change tar extraction from -xvzf
to -xzf to suppress verbose output. Replace rm output.tar.gz with rm -f
output.tar.gz to avoid errors if the file is missing. Additionally, add a
checksum verification step after download to ensure file integrity before
extraction.

perf:
deps:
- "download-dataset"
cmds:
- task: "package"
vars:
BUILD_WITH_SYMBOLS: true
- "rm -f perf.data"
- >-
perf record -F 99 -g -- ./build/core/clp-s c --timestamp-key "timestamp"
--target-encoded-size 268435456 build/clp_cockroachdb_perf_output {{.G_DATASET_LOCATION}}
- "chmod +777 perf.data"
- >-
perf report -i perf.data --call-graph=graph,0.5,0,caller,function,percent --stdio --demangle
--no-inline > perf-report.txt

init:
internal: true
silent: true
Expand Down