Merge pull request #489 from ziglang/mirrors

Mirrors!

Author: mlugg

.github/workflows/check-mirrors.yml

@@ -0,0 +1,28 @@
+name: Check Mirrors
+on:
+  schedule: [{ cron: "0 22 * * *" }]
+  workflow_dispatch: null
+  pull_request: { paths: ["assets/community-mirrors.ziggy"] }
+jobs:
+  check:
+    runs-on: [self-hosted, website]
+    steps:
+      - uses: actions/checkout@v2
+      - name: Check Mirrors
+        run: |
+          cd check-mirrors
+          /home/ci/deps/zig-linux-x86_64-0.14.0/zig build run -- ../assets/community-mirrors.ziggy "$GITHUB_STEP_SUMMARY"
+  notify-failure:
+    runs-on: [self-hosted, website]
+    needs: [check]
+    if: ${{ always() && (needs.check.result == 'failure' || needs.check.result == 'timed_out') }}
+    steps:
+      - name: Send Notification
+        env:
+          PUSHOVER_USER: ${{ secrets.MIRROR_CHECK_PUSHOVER_USER }}
+          PUSHOVER_TOKEN: ${{ secrets.MIRROR_CHECK_PUSHOVER_TOKEN }}
+        run: curl -s
+               -F user="$PUSHOVER_USER"
+               -F token="$PUSHOVER_TOKEN"
+               -F message="Zig download mirror validation failed"
+               https://api.pushover.net/1/messages.json

MIRRORS.md

@@ -0,0 +1,95 @@
+> [!NOTE]
+> Are you trying to download Zig from a mirror? If so, you're in the wrong place! This documentation is for anyone who wants to *host* a community mirror.
+> For instructions on *using* the mirrors, check out the [Community Mirrors](https://ziglang.org/download/community-mirrors/) page on the Zig website.
+
+# Community Mirrors
+
+The Zig mirrors in this repository are all community-maintained and unaffiliated with the ZSF. Anyone is welcome to host their own mirror and add it to the
+list. Because Zig's tarballs are cryptographically signed, no mirror need be trusted. Of course, if a mirror is found to be malicious, it will be removed from
+the list.
+
+The following rules define how a mirror works and the requirements it is expected to fulfil.
+
+* A mirror provides a single base URL, which we will call `X`.
+* `X` **may** include a path component, but is not required to. For instance, `https://foo.bar/zig/` is okay,
+  as is `https://zig.baz.qux/`.
+* The mirror **must** support HTTPS with a valid signed certificate. `X` **must** start with `https://`.
+* The mirror **must** cache tarballs locally. For instance, it may not simply forward all requests to another mirror.
+* The mirror **may** routinely evict its local tarball caches based on any reasonable factor, such as age, access frequency, or the existence of newer versions. This does not affect whether the mirror may respond with 404 Not Found for requests to these files (see below).
+* The mirror **must** download tarballs from `https://ziglang.org/` or a valid mirror of it.
+  * Typically, it is best to download only from `https://ziglang.org/`. One possible exception is when a pre-release is requested which is more likely to be available from an alternative mirror.
+* When a mirror receives a GET request for `X/<filename>`, the behavior depends on `<filename>`.
+  * Parse the file name to extract the Zig version string. More details on this are [below](#parsing-versions).
+  * Determine whether this is a "normal" or "pre-release" [Semantic Version](https://semver.org/).
+  * If this is a "normal" version, respond with 200 OK and the file found at `https://ziglang.org/download/<version>/<filename>`.
+    * If the release version is "0.5.0" or older (according to Semantic Versioning), the mirror **may** respond with 404 Not Found, but is not required to.
+    * Otherwise, the mirror **must** return the tarball.
+    * These requirements apply equally to source tarballs (e.g. `zig-0.14.1.tar.xz`), bootstrap source tarballs (e.g. `zig-bootstrap-0.14.1.tar.xz`), and binary tarballs (e.g. `zig-x86_64-linux-0.14.1.tar.xz`).
+  * Otherwise (i.e. if this is a "pre-release" version), respond with 200 OK and the file found at `https://ziglang.org/builds/<filename>`.
+    * If the version is older than the latest "release" version (according to Semantic Versioning), the mirror **may** respond with 404 Not Found, but is not required to.
+    * Otherwise, the mirror **must** return the tarball.
+    * These requirements apply equally to source tarballs (e.g. `zig-0.15.0-dev.671+c907866d5.tar.xz`), bootstrap source tarballs (e.g. `zig-bootstrap-0.15.0-dev.671+c907866d5.tar.xz`), and binary tarballs (e.g. `zig-x86_64-linux-0.15.0-dev.671+c907866d5.tar.xz`).
+  * Invalid accesses, for instance to malformed filenames, **may** cause a 404 Not Found response.
+  * Accesses to file names which do not end with ".zip", ".tar.xz", ".zip.minisig", or ".tar.xz.minisig", **may** cause a 404 Not Found response.
+* Files provided by the mirror **must** be bit-for-bit identical to their `https://ziglang.org/` counterparts.
+* If a mirror is required to serve a tarball which is has not yet cached locally, it **must** immediately download it from its source at `https://ziglang.org`, and respond with that downloaded file.
+* The mirror **may** rate-limit accesses. If an access failed due to rate-limiting, the mirror **should** respond with 429 Too Many Requests.
+* The mirror **may** undergo maintenance, upgrades, and other scheduled downtime. If an access fails for this reason, where possible, the mirror **should** respond with 503 Unavailable. The mirror **should** try to minimize such downtime.
+* The mirror **may** undergo occasional unintended and unscheduled downtime. The mirror **must** go to all efforts to minimize such outages, and **must** resolve such outages within a reasonable time upon being notified of them.
+* The mirror **may** observe the query parameter named `source` to learn about the origin of its traffic.
+  * Clients are encouraged, but not required, to provide this query parameter to indicate what service triggered the request. For instance, the `mlugg/setup-zig` GitHub Action passes this query parameter as `?source=github-mlugg-setup-zig`.
+
+## Parsing Versions
+
+Mirrors are required to parse tarball file names to extract the Zig version from them. Unfortunately,
+this is complicated a little by two factors: the existence of "source" tarballs, and the fact that
+Zig's tarball naming schema changed with the 0.14.1 release.
+
+If you have access to Regular Expressions, a simple strategy is to search the tarball name for
+`/\d+\.\d+\.\d+(-dev\.\d+\+[0-9a-f]+)?/`. This is probably inefficient, but that is unlikely to
+matter in practice.
+
+An alternative strategy which is simpler but requires slightly more code is as follows:
+* Strip a supported extension (`.zip`, `.tar.xz`, `.zip.minisig`, `.tar.xz.minisig`) from the end of the file name.
+* Find the last occurrence of "-" in the file name. If that byte is followed by the string "dev", find the previous occurence of "-" instead.
+* The string after that "-" byte is the Zig version.
+
+It is strongly discouraged for checks to rely on specific architecture or OS names, since the set of targets for which tarballs are provided could be extended at any time.
+
+## Adding A Mirror
+
+Once a mirror complies with the requirements listed above, you can add it to the list of mirrors on ziglang.org by modifying the list in
+`assets/community-mirrors.ziggy` and opening a pull request with your changes. The diff should just add a single entry to the end of the list:
+
+```diff
+ [
+     {
+         .url = "https://a.com",
+         .username = "a",
+         .email = "a@a.com",
+     },
+     {
+         .url = "https://b.com/zig",
+         .username = "b",
+         .email = "b@b.com",
+     },
++    {
++        .url = "https://mymirror.net",
++        .username = "my-github-username",
++        .email = "my@email.com",
++    },
+ ]
+```
+
+Note that the Zig core team always reserves the right to exclude mirrors from this list for any (or no) reason.
+
+## Community-Written Solutions
+
+Some members of the Zig community have written software for hosting Zig mirrors, making it easier to comply with the requirements listed above. If you have an
+open-source repository which would fit here, feel free to open a pull request adding it below.
+
+> [!WARNING]
+> The Zig Software Foundation has not written or audited this code. As such, no guarantees can be made regarding its correctness, and it should **not** be
+> implicitly trusted. You are strongly encouraged to audit this software before using it, particularly if it will not be run in an isolated environment.
+
+* [hexops/wrench](https://github.com/hexops/wrench?tab=readme-ov-file#run-your-own-ziglangorgdownload-mirror)

assets/community-mirrors.ziggy

@@ -0,0 +1,17 @@
+[
+    {
+        .url = "https://pkg.machengine.org/zig",
+        .username = "emidoots",
+        .email = "emi@hexops.com",
+    },
+    {
+        .url = "https://zigmirror.hryx.net/zig",
+        .username = "hryx",
+        .email = "codroid@gmail.com",
+    },
+    {
+        .url = "https://zig.linus.dev/zig",
+        .username = "linusg",
+        .email = "mail@linusgroh.de",
+    },
+]

assets/css/style.css

@@ -467,7 +467,7 @@ blockquote {
 }
 
 
-code.zig {
+code {
   --light-yellow: #e5c07b;
   --dark-yellow: #d19a66;
   --blue: #61afef;
@@ -517,3 +517,49 @@ video {
   margin-top: 1em;
   display: inline-block;
 }
+
+code.python {
+  color: var(--cyan);
+}
+
+code.python .punctuation {
+  color: var(--cyan);
+}
+
+code.python .string {
+  color: var(--dark-yellow);
+}
+
+code.python .variable,
+code.python .parameter,
+code.python .exception,
+code.python .field {
+  color: var(--light-yellow);
+}
+
+code.python .keyword.function {
+  color: var(--light-red);
+}
+
+code.python .bracket {
+  color: var(--cyan);
+}
+
+code.python .function {
+  color: var(--blue);
+}
+
+code.python .builtin {
+  color: var(--magenta);
+}
+
+code.python .number {
+  color: var(--magenta);
+}
+
+code.python .operator,
+code.python .qualifier,
+code.python .keyword,
+code.python .attribute {
+  color: var(--light-red);
+}

build.zig.zon

@@ -13,8 +13,8 @@
             .hash = "doctest-0.1.0-lY-6k0W2AQB2cw2vD3306oCziQrVnG013naZd8hMkuAQ",
         },
         .zine = .{
-            .url = "git+https://github.com/kristoff-it/zine#5408d3dff6107fc449a6d24d90f379b6fffdb4e6",
-            .hash = "zine-0.9.0-ou6nIBcqFgCkkL8OQnHRzEKboVcAvIp76bnf6n9qeBvU",
+            .url = "git+https://github.com/kristoff-it/zine#ff0b7b1222fe6529fc36e876a71a99379ad761c4",
+            .hash = "zine-0.9.0-ou6nIEJTFgDwdlQnB6NtTnZneWKgg-qoVPaXyU2PjWF2",
         },
     },
 }

check-mirrors/README.md

@@ -0,0 +1,8 @@
+# check-mirrors
+
+This is a small utility which checks the functionality of all listed community mirrors. It does this
+by fetching a subset of available tarballs and signatures from each mirror, and checking that the
+returned files exactly match the corresponding files served by ziglang.org.
+
+This check is automatically run once per day by the 'check-mirrors' workflow. It also runs when a PR
+modifies the mirror list.

check-mirrors/build.zig

@@ -0,0 +1,28 @@
+pub fn build(b: *std.Build) void {
+    const target = b.standardTargetOptions(.{});
+    const optimize = b.standardOptimizeOption(.{});
+
+    const ziggy = b.dependency("ziggy", .{
+        .target = target,
+        .optimize = optimize,
+    });
+
+    const exe = b.addExecutable(.{
+        .name = "check-mirrors",
+        .root_module = b.createModule(.{
+            .root_source_file = b.path("main.zig"),
+            .target = target,
+            .optimize = optimize,
+            .imports = &.{
+                .{ .name = "ziggy", .module = ziggy.module("ziggy") },
+            },
+        }),
+    });
+    b.installArtifact(exe);
+
+    const run = b.addRunArtifact(exe);
+    if (b.args) |args| run.addArgs(args);
+    b.step("run", "Run check-mirrors").dependOn(&run.step);
+}
+
+const std = @import("std");

check-mirrors/build.zig.zon

@@ -0,0 +1,17 @@
+.{
+    .name = .check_mirrors,
+    .version = "0.0.0",
+    .fingerprint = 0xcb408e1ecbee16fd, // Changing this has security and trust implications.
+    .minimum_zig_version = "0.14.0",
+    .dependencies = .{
+        .ziggy = .{
+            .url = "git+https://github.com/kristoff-it/ziggy.git#fe3bf9389e7ff213cf3548caaf9c6f3d4bb38647",
+            .hash = "ziggy-0.1.0-kTg8vwBEBgDFzbstERaPLr5TzM1DhfDza1we_eEXuLDL",
+        },
+    },
+    .paths = .{
+        "build.zig",
+        "build.zig.zon",
+        "src",
+    },
+}