Merge pull request #1904 from grumpycoders/archive

nicolasnoble · web-flow · commit 82587ee39524 · 2025-03-15T11:15:29.000-07:00
Adding psyqo's archive system.
diff --git a/src/mips/psyqo-paths/Makefile b/src/mips/psyqo-paths/Makefile
@@ -5,6 +5,8 @@ TYPE = library
 
 SRCS = \
 src/cdrom-loader.cpp \
+src/archive-manager.cpp \
+../ucl-demo/n2e-d.S \
 
 EXTRA_DEPS += $(PSYQOPATHSDIR)Makefile
 
diff --git a/src/mips/psyqo-paths/archive-manager.hh b/src/mips/psyqo-paths/archive-manager.hh
diff --git a/src/mips/psyqo-paths/cdrom-loader.hh b/src/mips/psyqo-paths/cdrom-loader.hh
@@ -27,12 +27,11 @@ SOFTWARE.
 #pragma once
 
 #include <EASTL/string_view.h>
-#include <EASTL/vector.h>
 #include <stdint.h>
 
 #include <coroutine>
 
-#include "psyqo/gpu.hh"
+#include "psyqo/buffer.hh"
 #include "psyqo/iso9660-parser.hh"
 #include "psyqo/task.hh"
 
@@ -44,7 +43,7 @@ namespace psyqo::paths {
  * @details This class provides a PSYQo path to read files from the CDRom.
  * The way to use it is to instantiate the class somewhere persistent, and
  * then call readFile() with a callback. The callback will be called with
- * the data of the file, or an empty vector if the file could not be read.
+ * the data of the file, or an empty buffer if the file could not be read.
  * This is going to allocate memory in different places. Only one file can
  * be read at a time, but it is safe to call readFile() again from the
  * callback. If preferred, the loader can be cascaded into another `TaskQueue`.
@@ -54,59 +53,70 @@ namespace psyqo::paths {
 
 class CDRomLoader {
     struct ReadFileAwaiter {
-        ReadFileAwaiter(eastl::string_view path, GPU &gpu, ISO9660Parser &parser, CDRomLoader &loader)
-            : m_path(path), m_gpu(gpu), m_parser(parser), m_loader(loader) {}
+        ReadFileAwaiter(eastl::string_view path, ISO9660Parser &parser, CDRomLoader &loader)
+            : m_path(path), m_parser(parser), m_loader(loader) {}
         ~ReadFileAwaiter() {}
         constexpr bool await_ready() const { return false; }
         template <typename U>
         void await_suspend(std::coroutine_handle<U> handle) {
-            m_loader.readFile(m_path, m_gpu, m_parser, [handle, this](eastl::vector<uint8_t> &&data) {
+            m_loader.readFile(m_path, m_parser, [handle, this](Buffer<uint8_t> &&data) {
                 m_data = eastl::move(data);
                 handle.resume();
             });
         }
-        eastl::vector<uint8_t> await_resume() { return eastl::move(m_data); }
+        Buffer<uint8_t> await_resume() { return eastl::move(m_data); }
 
       private:
         eastl::string_view m_path;
-        GPU &m_gpu;
         ISO9660Parser &m_parser;
         CDRomLoader &m_loader;
-        eastl::vector<uint8_t> m_data;
+        Buffer<uint8_t> m_data;
     };
 
   public:
+    /**
+     * @brief Set the Buffer object for the next read operation.
+     *
+     * @details This function sets the buffer to be used for the next read
+     * operation. By default, the archive manager will allocate a buffer of the
+     * appropriate size for the file being read. However, if the user wants to
+     * use an already allocated buffer, they can use this function to set the buffer
+     * to be used.
+     *
+     * @param buffer The buffer to be used for the next read operation.
+     */
+    void setBuffer(Buffer<uint8_t> &&buffer) { m_data = eastl::move(buffer); }
+
     /**
      * @brief Reads a file from the CDRom.
      *
      * @param path The path to the file to read. The view must be persistent
      * until the callback is called.
-     * @param gpu The GPU class used by the application, in order to set timers.
      * @param parser The ISO9660Parser to use for reading the file.
      * @param callback The callback to call when the file is read. The callback
-     * will be called with the data of the file, or an empty vector if the file
+     * will be called with the data of the file, or an empty buffer if the file
      * could not be read.
      */
-    void readFile(eastl::string_view path, GPU &gpu, ISO9660Parser &parser,
-                  eastl::function<void(eastl::vector<uint8_t> &&)> &&callback) {
-        setupQueue(path, gpu, parser, eastl::move(callback));
+    void readFile(eastl::string_view path, ISO9660Parser &parser,
+                  eastl::function<void(Buffer<uint8_t> &&)> &&callback) {
+        setupQueue(path, parser, eastl::move(callback));
         m_queue.run();
     }
-    psyqo::TaskQueue::Task scheduleReadFile(eastl::string_view path, GPU &gpu, ISO9660Parser &parser) {
-        setupQueue(path, gpu, parser, {});
+    psyqo::TaskQueue::Task scheduleReadFile(eastl::string_view path, ISO9660Parser &parser) {
+        setupQueue(path, parser, {});
         return m_queue.schedule();
     }
-    ReadFileAwaiter readFile(eastl::string_view path, GPU &gpu, ISO9660Parser &parser) {
-        return {path, gpu, parser, *this};
+    ReadFileAwaiter readFile(eastl::string_view path, ISO9660Parser &parser) {
+        return {path, parser, *this};
     }
 
   private:
-    void setupQueue(eastl::string_view path, GPU &gpu, ISO9660Parser &parser,
-                    eastl::function<void(eastl::vector<uint8_t> &&)> &&callback);
-    eastl::function<void(eastl::vector<uint8_t> &&)> m_callback;
+    void setupQueue(eastl::string_view path, ISO9660Parser &parser,
+                    eastl::function<void(Buffer<uint8_t> &&)> &&callback);
+    eastl::function<void(Buffer<uint8_t> &&)> m_callback;
     psyqo::TaskQueue m_queue;
     ISO9660Parser::ReadRequest m_request;
-    eastl::vector<uint8_t> m_data;
+    Buffer<uint8_t> m_data;
     bool m_pending = false;
 };
 
diff --git a/src/mips/psyqo-paths/examples/cdrom-loader/cdrom-loader.cpp b/src/mips/psyqo-paths/examples/cdrom-loader/cdrom-loader.cpp
@@ -44,7 +44,7 @@ class CDRomLoaderExample final : public psyqo::Application {
     psyqo::CDRomDevice m_cdrom;
     psyqo::ISO9660Parser m_isoParser = psyqo::ISO9660Parser(&m_cdrom);
     psyqo::paths::CDRomLoader m_cdromLoader;
-    eastl::vector<uint8_t> m_buffer;
+    psyqo::Buffer<uint8_t> m_buffer;
     bool m_callbackCalled = false;
 };
 
@@ -70,8 +70,8 @@ void CDRomLoaderExample::prepare() {
 void CDRomLoaderExample::createScene() {
     m_font.uploadSystemFont(gpu());
     pushScene(&cdromLoaderExampleScene);
-    m_cdromLoader.readFile("SYSTEM.CNF;1", cdromLoaderExample.gpu(), cdromLoaderExample.m_isoParser,
-                           [this](eastl::vector<uint8_t>&& buffer) {
+    m_cdromLoader.readFile("SYSTEM.CNF;1", cdromLoaderExample.m_isoParser,
+                           [this](psyqo::Buffer<uint8_t>&& buffer) {
                                m_buffer = eastl::move(buffer);
                                m_callbackCalled = true;
                            });
diff --git a/src/mips/psyqo-paths/src/archive-manager.cpp b/src/mips/psyqo-paths/src/archive-manager.cpp
@@ -0,0 +1,165 @@
+/*
+
+MIT License
+
+Copyright (c) 2025 PCSX-Redux authors
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.
+
+*/
+
+#include "psyqo-paths/archive-manager.hh"
+
+#include "EASTL/algorithm.h"
+#include "lz4/lz4.h"
+#include "psyqo/kernel.hh"
+#include "psyqo/utility-polyfill.h"
+#include "ucl-demo/n2e-d.h"
+
+eastl::array<void (psyqo::paths::ArchiveManager::*)(const psyqo::paths::ArchiveManager::IndexEntry*), 3>
+    psyqo::paths::ArchiveManager::s_decompressors = {nullptr};
+
+void psyqo::paths::ArchiveManager::setupInitQueue(eastl::string_view archiveName, ISO9660Parser& parser,
+                                                  eastl::function<void(bool)>&& callback) {
+    setupInitQueue(0, *parser.getCDRom(), eastl::move(callback));
+    m_queueInitFilename.reset();
+    m_queueInitFilename.startWith([](auto task) { task->resolve(); });
+    if (!parser.initialized()) {
+        m_queueInitFilename.then(parser.scheduleInitialize());
+    }
+    m_queueInitFilename.then(parser.scheduleGetDirentry(archiveName, &m_archiveDirentry))
+        .then([this](auto task) {
+            m_index.resize(2048 / sizeof(IndexEntry));
+            m_request.LBA = m_archiveDirentry.LBA;
+            m_request.count = 1;
+            m_request.buffer = m_index.data();
+            task->resolve();
+        })
+        .then(m_queue.schedule());
+}
+
+void psyqo::paths::ArchiveManager::setupInitQueue(uint32_t LBA, CDRom& device, eastl::function<void(bool)>&& callback) {
+    Kernel::assert(!m_pending, "Only one action can be performed at a time");
+    m_queue.reset();
+    m_pending = true;
+    m_success = false;
+    m_initCallback = eastl::move(callback);
+    m_request.LBA = LBA;
+    m_request.count = 1;
+    m_request.buffer = m_index.data();
+    m_queue.startWith(device.scheduleReadRequest(&m_request))
+        .then([this](auto task) {
+            eastl::string_view signature = {reinterpret_cast<const char*>(m_index.data()), 8};
+            if (signature != "PSX-ARC1") {
+                task->reject();
+                return;
+            }
+            m_index.resize(getIndexSectorCount() * 2048 / sizeof(IndexEntry));
+            m_request.count = getIndexSectorCount();
+            m_request.buffer = m_index.data();
+            task->resolve();
+        })
+        .then(device.scheduleReadRequest(&m_request))
+        .then([this](auto task) {
+            m_success = true;
+            m_index.resize(getIndexCount() + 1);
+            task->resolve();
+        })
+        .finally([this](auto queue) {
+            m_pending = false;
+            auto callback = eastl::move(m_initCallback);
+            m_initCallback = nullptr;
+            callback(m_success);
+        });
+}
+
+const psyqo::paths::ArchiveManager::IndexEntry* psyqo::paths::ArchiveManager::getIndexEntry(
+    eastl::string_view path) const {
+    uint64_t hash = djb::hash<uint64_t>(path.data(), path.size());
+    return getIndexEntry(hash);
+}
+
+const psyqo::paths::ArchiveManager::IndexEntry* psyqo::paths::ArchiveManager::getIndexEntry(uint64_t hash) const {
+    const IndexEntry* first = &m_index[1];
+    const IndexEntry* last = first + getIndexCount();
+    const IndexEntry* entry =
+        eastl::lower_bound(first, last, hash, [](const IndexEntry& e, uint64_t hash) { return e.hash < hash; });
+    if (entry != last && entry->hash == hash) {
+        return entry;
+    }
+    return nullptr;
+}
+
+void psyqo::paths::ArchiveManager::setupQueue(const IndexEntry* entry, CDRom& device,
+                                              eastl::function<void(Buffer<uint8_t>&&)>&& callback) {
+    Kernel::assert(!m_pending, "Only one action can be performed at a time");
+    m_queue.reset();
+    m_pending = true;
+    m_callback = eastl::move(callback);
+    if (!entry) {
+        m_queue.startWith([this](auto task) {
+            m_data.clear();
+            m_pending = false;
+            auto callback = eastl::move(m_callback);
+            m_callback = nullptr;
+            callback(eastl::move(m_data));
+        });
+        return;
+    }
+    const auto method = entry->getCompressionMethod();
+    const uint32_t sectorCount = entry->getCompressedSize();
+    const uint32_t decompSize = entry->getDecompSize();
+    m_request.LBA = m_archiveDirentry.LBA + entry->getSectorOffset();
+    m_request.count = sectorCount;
+    if (method == IndexEntry::Method::NONE) {
+        m_data.resize(sectorCount * 2048);
+        m_request.buffer = m_data.data();
+    } else {
+        uint32_t actualSize = eastl::max<uint32_t>(((decompSize + 3) & ~3) + 16, sectorCount * 2048);
+        m_data.resize(actualSize);
+        m_request.buffer = m_data.data() + actualSize - sectorCount * 2048;
+    }
+    m_queue.startWith(device.scheduleReadRequest(&m_request))
+        .then([this, entry](auto task) {
+            auto decompress = s_decompressors[toUnderlying(entry->getCompressionMethod())];
+            if (decompress) (this->*decompress)(entry);
+            uint32_t decompSize = entry->getDecompSize();
+            if (decompSize) m_data.resize(decompSize);
+            task->resolve();
+        })
+        .butCatch([this](auto task) { m_data.resize(0); })
+        .finally([this, decompSize](auto queue) {
+            m_pending = false;
+            auto callback = eastl::move(m_callback);
+            m_callback = nullptr;
+            callback(eastl::move(m_data));
+        });
+}
+
+void psyqo::paths::ArchiveManager::decompressUCL_NRV2E(const IndexEntry* entry) {
+    uint32_t padding = entry->getPadding();
+    n2e_decompress(reinterpret_cast<uint8_t*>(m_request.buffer) + padding, m_data.data());
+}
+
+void psyqo::paths::ArchiveManager::decompressLZ4(const IndexEntry* entry) {
+    uint32_t padding = entry->getPadding();
+    uint32_t srcSize = entry->getCompressedSize() * 2048 - padding;
+    uint8_t* src = reinterpret_cast<uint8_t*>(m_request.buffer) + padding;
+    lz4_decompress_block(src, src + srcSize, m_data.data());
+}
diff --git a/src/mips/psyqo-paths/src/cdrom-loader.cpp b/src/mips/psyqo-paths/src/cdrom-loader.cpp
@@ -28,8 +28,8 @@ SOFTWARE.
 
 #include "psyqo/kernel.hh"
 
-void psyqo::paths::CDRomLoader::setupQueue(eastl::string_view path, GPU& gpu, psyqo::ISO9660Parser& parser,
-                                           eastl::function<void(eastl::vector<uint8_t>&&)>&& callback) {
+void psyqo::paths::CDRomLoader::setupQueue(eastl::string_view path, psyqo::ISO9660Parser& parser,
+                                           eastl::function<void(Buffer<uint8_t>&&)>&& callback) {
     Kernel::assert(!m_pending, "Only one file can be read at a time");
     m_queue.reset();
     m_pending = true;
diff --git a/src/mips/psyqo-paths/tools/mkarchive.lua b/src/mips/psyqo-paths/tools/mkarchive.lua
